Visual Servoing Platform  version 3.1.0
testKeyPoint-4.cpp
1 /****************************************************************************
2  *
3  * This file is part of the ViSP software.
4  * Copyright (C) 2005 - 2017 by Inria. All rights reserved.
5  *
6  * This software is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  * See the file LICENSE.txt at the root directory of this source
11  * distribution for additional information about the GNU GPL.
12  *
13  * For using ViSP with software that can not be combined with the GNU
14  * GPL, please contact Inria about acquiring a ViSP Professional
15  * Edition License.
16  *
17  * See http://visp.inria.fr for more information.
18  *
19  * This software was developed at:
20  * Inria Rennes - Bretagne Atlantique
21  * Campus Universitaire de Beaulieu
22  * 35042 Rennes Cedex
23  * France
24  *
25  * If you have questions regarding the use of this file, please contact
26  * Inria at visp@inria.fr
27  *
28  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
29  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
30  *
31  * Description:
32  * Test keypoint matching and pose estimation with mostly OpenCV functions
33  *calls to detect potential memory leaks in testKeyPoint-2.cpp.
34  *
35  * Authors:
36  * Souriya Trinh
37  *
38  *****************************************************************************/
39 
40 #include <iostream>
41 
42 #include <visp3/core/vpConfig.h>
43 
44 #if defined(VISP_HAVE_OPENCV) && (VISP_HAVE_OPENCV_VERSION >= 0x020301)
45 
46 #include <opencv2/core/core.hpp>
47 #include <opencv2/features2d/features2d.hpp>
48 #include <visp3/core/vpHomogeneousMatrix.h>
49 #include <visp3/core/vpImage.h>
50 #include <visp3/core/vpIoTools.h>
51 #include <visp3/gui/vpDisplayGDI.h>
52 #include <visp3/gui/vpDisplayGTK.h>
53 #include <visp3/gui/vpDisplayOpenCV.h>
54 #include <visp3/gui/vpDisplayX.h>
55 #include <visp3/io/vpImageIo.h>
56 #include <visp3/io/vpParseArgv.h>
57 #include <visp3/io/vpVideoReader.h>
58 #include <visp3/mbt/vpMbEdgeTracker.h>
59 #include <visp3/vision/vpKeyPoint.h>
60 
61 // List of allowed command line options
62 #define GETOPTARGS "cdh"
63 
64 void usage(const char *name, const char *badparam);
65 bool getOptions(int argc, const char **argv, bool &click_allowed, bool &display);
66 
75 void usage(const char *name, const char *badparam)
76 {
77  fprintf(stdout, "\n\
78 Test keypoints matching.\n\
79 \n\
80 SYNOPSIS\n\
81  %s [-c] [-d] [-h]\n", name);
82 
83  fprintf(stdout, "\n\
84 OPTIONS: \n\
85 \n\
86  -c\n\
87  Disable the mouse click. Useful to automate the \n\
88  execution of this program without human intervention.\n\
89 \n\
90  -d \n\
91  Turn off the display.\n\
92 \n\
93  -h\n\
94  Print the help.\n");
95 
96  if (badparam)
97  fprintf(stdout, "\nERROR: Bad parameter [%s]\n", badparam);
98 }
99 
111 bool getOptions(int argc, const char **argv, bool &click_allowed, bool &display)
112 {
113  const char *optarg_;
114  int c;
115  while ((c = vpParseArgv::parse(argc, argv, GETOPTARGS, &optarg_)) > 1) {
116 
117  switch (c) {
118  case 'c':
119  click_allowed = false;
120  break;
121  case 'd':
122  display = false;
123  break;
124  case 'h':
125  usage(argv[0], NULL);
126  return false;
127  break;
128 
129  default:
130  usage(argv[0], optarg_);
131  return false;
132  break;
133  }
134  }
135 
136  if ((c == 1) || (c == -1)) {
137  // standalone param or error
138  usage(argv[0], NULL);
139  std::cerr << "ERROR: " << std::endl;
140  std::cerr << " Bad argument " << optarg_ << std::endl << std::endl;
141  return false;
142  }
143 
144  return true;
145 }
146 
153 int main(int argc, const char **argv)
154 {
155  try {
156  std::string env_ipath;
157  bool opt_click_allowed = true;
158  bool opt_display = true;
159 
160  // Read the command line options
161  if (getOptions(argc, argv, opt_click_allowed, opt_display) == false) {
162  exit(-1);
163  }
164 
165  // Get the visp-images-data package path or VISP_INPUT_IMAGE_PATH
166  // environment variable value
167  env_ipath = vpIoTools::getViSPImagesDataPath();
168 
169  if (env_ipath.empty()) {
170  std::cerr << "Please set the VISP_INPUT_IMAGE_PATH environment "
171  "variable value."
172  << std::endl;
173  return -1;
174  }
175 
176  vpImage<unsigned char> I, Imatch, Iref;
177 
178  // Set the path location of the image sequence
179  std::string dirname = vpIoTools::createFilePath(env_ipath, "mbt/cube");
180 
181  // Build the name of the image files
182  std::string filenameRef = vpIoTools::createFilePath(dirname, "image0000.pgm");
183  vpImageIo::read(I, filenameRef);
184  Iref = I;
185  std::string filenameCur = vpIoTools::createFilePath(dirname, "image%04d.pgm");
186 
187 #if defined VISP_HAVE_X11
188  vpDisplayX display, display2;
189 #elif defined VISP_HAVE_GTK
190  vpDisplayGTK display, display2;
191 #elif defined VISP_HAVE_GDI
192  vpDisplayGDI display, display2;
193 #else
194  vpDisplayOpenCV display, display2;
195 #endif
196 
197  if (opt_display) {
199  display.init(I, 0, 0, "ORB keypoints matching");
200  Imatch.resize(I.getHeight(), 2 * I.getWidth());
201  Imatch.insert(I, vpImagePoint(0, 0));
203  display2.init(Imatch, 0, (int)I.getHeight() / vpDisplay::getDownScalingFactor(I) + 70, "ORB keypoints matching");
204  }
205 
206  vpCameraParameters cam;
207  vpMbEdgeTracker tracker;
208  // Load config for tracker
209  std::string tracker_config_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.xml");
210 
211 #ifdef VISP_HAVE_XML2
212  tracker.loadConfigFile(tracker_config_file);
213  tracker.getCameraParameters(cam);
214 #else
215  vpMe me;
216  me.setMaskSize(5);
217  me.setMaskNumber(180);
218  me.setRange(8);
219  me.setThreshold(10000);
220  me.setMu1(0.5);
221  me.setMu2(0.5);
222  me.setSampleStep(4);
223  me.setNbTotalSample(250);
224  tracker.setMovingEdge(me);
225  cam.initPersProjWithoutDistortion(547.7367575, 542.0744058, 338.7036994, 234.5083345);
226  tracker.setCameraParameters(cam);
227  tracker.setNearClippingDistance(0.01);
228  tracker.setFarClippingDistance(100.0);
230 #endif
231 
232  tracker.setAngleAppear(vpMath::rad(89));
233  tracker.setAngleDisappear(vpMath::rad(89));
234 
235  // Load CAO model
236  std::string cao_model_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.cao");
237  tracker.loadModel(cao_model_file);
238 
239  // Initialize the pose
240  std::string init_file = vpIoTools::createFilePath(env_ipath, "mbt/cube.init");
241  if (opt_display && opt_click_allowed) {
242  tracker.initClick(I, init_file);
243  } else {
244  vpHomogeneousMatrix cMoi(0.02044769891, 0.1101505452, 0.5078963719, 2.063603907, 1.110231561, -0.4392789872);
245  tracker.initFromPose(I, cMoi);
246  }
247 
248  // Get the init pose
250  tracker.getPose(cMo);
251 
252  // Init keypoints
253  cv::Ptr<cv::FeatureDetector> detector;
254  cv::Ptr<cv::DescriptorExtractor> extractor;
255  cv::Ptr<cv::DescriptorMatcher> matcher;
256 
257 #if (VISP_HAVE_OPENCV_VERSION >= 0x030000)
258  detector = cv::ORB::create(500, 1.2f, 1);
259  extractor = cv::ORB::create(500, 1.2f, 1);
260 #elif (VISP_HAVE_OPENCV_VERSION >= 0x020301)
261  detector = cv::FeatureDetector::create("ORB");
262  extractor = cv::DescriptorExtractor::create("ORB");
263 #endif
264  matcher = cv::DescriptorMatcher::create("BruteForce-Hamming");
265 
266 #if (VISP_HAVE_OPENCV_VERSION >= 0x020400 && VISP_HAVE_OPENCV_VERSION < 0x030000)
267  detector->set("nLevels", 1);
268 #endif
269 
270  // Detect keypoints on the current image
271  std::vector<cv::KeyPoint> trainKeyPoints;
272  cv::Mat matImg;
273  vpImageConvert::convert(I, matImg);
274  detector->detect(matImg, trainKeyPoints);
275 
276  // Keep only keypoints on the cube
277  std::vector<vpPolygon> polygons;
278  std::vector<std::vector<vpPoint> > roisPt;
279  std::pair<std::vector<vpPolygon>, std::vector<std::vector<vpPoint> > > pair = tracker.getPolygonFaces(false);
280  polygons = pair.first;
281  roisPt = pair.second;
282 
283  // Compute the 3D coordinates
284  std::vector<cv::Point3f> points3f;
285  vpKeyPoint::compute3DForPointsInPolygons(cMo, cam, trainKeyPoints, polygons, roisPt, points3f);
286 
287  // Extract descriptors
288  cv::Mat trainDescriptors;
289  extractor->compute(matImg, trainKeyPoints, trainDescriptors);
290 
291  if (trainKeyPoints.size() != (size_t)trainDescriptors.rows || trainKeyPoints.size() != points3f.size()) {
292  std::cerr << "Problem with training data size !" << std::endl;
293  return -1;
294  }
295 
296  // Init reader for getting the input image sequence
297  vpVideoReader g;
298  g.setFileName(filenameCur);
299  g.open(I);
300  g.acquire(I);
301 
302  bool opt_click = false;
304  while ((opt_display && !g.end()) || (!opt_display && g.getFrameIndex() < 30)) {
305  g.acquire(I);
306 
307  vpImageConvert::convert(I, matImg);
308  std::vector<cv::KeyPoint> queryKeyPoints;
309  detector->detect(matImg, queryKeyPoints);
310 
311  cv::Mat queryDescriptors;
312  extractor->compute(matImg, queryKeyPoints, queryDescriptors);
313 
314  std::vector<std::vector<cv::DMatch> > knn_matches;
315  std::vector<cv::DMatch> matches;
316  matcher->knnMatch(queryDescriptors, trainDescriptors, knn_matches, 2);
317  for (std::vector<std::vector<cv::DMatch> >::const_iterator it = knn_matches.begin(); it != knn_matches.end();
318  ++it) {
319  if (it->size() > 1) {
320  double ratio = (*it)[0].distance / (*it)[1].distance;
321  if (ratio < 0.85) {
322  matches.push_back((*it)[0]);
323  }
324  }
325  }
326 
327  vpPose estimated_pose;
328  for (std::vector<cv::DMatch>::const_iterator it = matches.begin(); it != matches.end(); ++it) {
329  vpPoint pt(points3f[(size_t)(it->trainIdx)].x, points3f[(size_t)(it->trainIdx)].y,
330  points3f[(size_t)(it->trainIdx)].z);
331 
332  double x = 0.0, y = 0.0;
333  vpPixelMeterConversion::convertPoint(cam, queryKeyPoints[(size_t)(it->queryIdx)].pt.x,
334  queryKeyPoints[(size_t)(it->queryIdx)].pt.y, x, y);
335  pt.set_x(x);
336  pt.set_y(y);
337 
338  estimated_pose.addPoint(pt);
339  }
340 
341  bool is_pose_estimated = false;
342  if (estimated_pose.npt >= 4) {
343  try {
344  unsigned int nb_inliers = (unsigned int)(0.6 * estimated_pose.npt);
345  estimated_pose.setRansacNbInliersToReachConsensus(nb_inliers);
346  estimated_pose.setRansacThreshold(0.01);
347  estimated_pose.setRansacMaxTrials(500);
348  estimated_pose.computePose(vpPose::RANSAC, cMo);
349  is_pose_estimated = true;
350  } catch (...) {
351  is_pose_estimated = false;
352  }
353  }
354 
355  if (opt_display) {
357 
358  Imatch.insert(I, vpImagePoint(0, Iref.getWidth()));
359  vpDisplay::display(Imatch);
360  for (std::vector<cv::DMatch>::const_iterator it = matches.begin(); it != matches.end(); ++it) {
361  vpImagePoint leftPt(trainKeyPoints[(size_t)it->trainIdx].pt.y, trainKeyPoints[(size_t)it->trainIdx].pt.x);
362  vpImagePoint rightPt(queryKeyPoints[(size_t)it->queryIdx].pt.y,
363  queryKeyPoints[(size_t)it->queryIdx].pt.x + Iref.getWidth());
364  vpDisplay::displayLine(Imatch, leftPt, rightPt, vpColor::green);
365  }
366 
367  if (is_pose_estimated) {
368  tracker.setPose(I, cMo);
369  tracker.display(I, cMo, cam, vpColor::red);
370  vpDisplay::displayFrame(I, cMo, cam, 0.05, vpColor::none);
371  }
372 
373  vpDisplay::flush(Imatch);
374  vpDisplay::flush(I);
375  }
376 
377  // Click requested to process next image
378  if (opt_click_allowed && opt_display) {
379  if (opt_click) {
380  vpDisplay::getClick(I, button, true);
381  if (button == vpMouseButton::button3) {
382  opt_click = false;
383  }
384  } else {
385  // Use right click to enable/disable step by step tracking
386  if (vpDisplay::getClick(I, button, false)) {
387  if (button == vpMouseButton::button3) {
388  opt_click = true;
389  } else if (button == vpMouseButton::button1) {
390  break;
391  }
392  }
393  }
394  }
395  }
396 
397  } catch (vpException &e) {
398  std::cerr << e.what() << std::endl;
399  return -1;
400  }
401 
402  std::cout << "testKeyPoint-4 is ok !" << std::endl;
403  return 0;
404 }
405 #else
406 int main()
407 {
408  std::cerr << "You need OpenCV library." << std::endl;
409 
410  return 0;
411 }
412 
413 #endif
virtual unsigned int getClipping() const
Definition: vpMbTracker.h:223
void setMovingEdge(const vpMe &me)
virtual void getPose(vpHomogeneousMatrix &cMo_) const
Definition: vpMbTracker.h:381
static bool getClick(const vpImage< unsigned char > &I, bool blocking=true)
static std::string getViSPImagesDataPath()
Definition: vpIoTools.cpp:1214
virtual void setAngleDisappear(const double &a)
Definition: vpMbTracker.h:433
static void convert(const vpImage< unsigned char > &src, vpImage< vpRGBa > &dest)
Implementation of an homogeneous matrix and operations on such kind of matrices.
void setMaskNumber(const unsigned int &a)
Definition: vpMe.cpp:454
virtual void setDownScalingFactor(unsigned int scale)
Definition: vpDisplay.cpp:232
Display for windows using GDI (available on any windows 32 platform).
Definition: vpDisplayGDI.h:129
void setSampleStep(const double &s)
Definition: vpMe.h:278
void setNbTotalSample(const int &nb)
Definition: vpMe.h:255
Use the X11 console to display images on unix-like OS. Thus to enable this class X11 should be instal...
Definition: vpDisplayX.h:151
Class that enables to manipulate easily a video file or a sequence of images. As it inherits from the...
static const vpColor none
Definition: vpColor.h:192
error that can be emited by ViSP classes.
Definition: vpException.h:71
void setRansacThreshold(const double &t)
Definition: vpPose.h:252
void init(vpImage< unsigned char > &I, int winx=-1, int winy=-1, const std::string &title="")
Definition: vpMe.h:60
static void convertPoint(const vpCameraParameters &cam, const double &u, const double &v, double &x, double &y)
Point coordinates conversion from pixel coordinates to normalized coordinates in meter...
Make the complete tracking of an object by using its CAD model.
virtual void setPose(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cdMo)
virtual void setCameraParameters(const vpCameraParameters &camera)
virtual void initFromPose(const vpImage< unsigned char > &I, const std::string &initFile)
void loadConfigFile(const std::string &configFile)
static const vpColor green
Definition: vpColor.h:183
static void flush(const vpImage< unsigned char > &I)
void setMu1(const double &mu_1)
Definition: vpMe.h:241
static bool parse(int *argcPtr, const char **argv, vpArgvInfo *argTable, int flags)
Definition: vpParseArgv.cpp:69
static const vpColor red
Definition: vpColor.h:180
Class that defines what is a point.
Definition: vpPoint.h:58
void initPersProjWithoutDistortion(const double px, const double py, const double u0, const double v0)
virtual void setNearClippingDistance(const double &dist)
void open(vpImage< vpRGBa > &I)
static void compute3DForPointsInPolygons(const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, std::vector< cv::KeyPoint > &candidates, const std::vector< vpPolygon > &polygons, const std::vector< std::vector< vpPoint > > &roisPt, std::vector< cv::Point3f > &points, cv::Mat *descriptors=NULL)
Definition: vpKeyPoint.cpp:762
static std::string createFilePath(const std::string &parent, const std::string &child)
Definition: vpIoTools.cpp:1439
void setMaskSize(const unsigned int &a)
Definition: vpMe.cpp:461
static void display(const vpImage< unsigned char > &I)
The vpDisplayOpenCV allows to display image using the OpenCV library. Thus to enable this class OpenC...
Class used for pose computation from N points (pose from point only). Some of the algorithms implemen...
Definition: vpPose.h:79
Generic class defining intrinsic camera parameters.
The vpDisplayGTK allows to display image using the GTK 3rd party library. Thus to enable this class G...
Definition: vpDisplayGTK.h:138
void acquire(vpImage< vpRGBa > &I)
bool computePose(vpPoseMethodType method, vpHomogeneousMatrix &cMo, bool(*func)(vpHomogeneousMatrix *)=NULL)
Definition: vpPose.cpp:362
void resize(const unsigned int h, const unsigned int w)
resize the image : Image initialization
Definition: vpImage.h:856
virtual void setFarClippingDistance(const double &dist)
void setFileName(const char *filename)
virtual void setAngleAppear(const double &a)
Definition: vpMbTracker.h:422
virtual void initClick(const vpImage< unsigned char > &I, const std::string &initFile, const bool displayHelp=false)
void insert(const vpImage< Type > &src, const vpImagePoint &topLeft)
Definition: vpImage.h:1102
const char * what() const
static double rad(double deg)
Definition: vpMath.h:102
unsigned int npt
Number of point used in pose computation.
Definition: vpPose.h:114
void setRansacMaxTrials(const int &rM)
Definition: vpPose.h:261
void setRansacNbInliersToReachConsensus(const unsigned int &nbC)
Definition: vpPose.h:251
void setMu2(const double &mu_2)
Definition: vpMe.h:248
long getFrameIndex() const
virtual void loadModel(const char *modelFile, const bool verbose=false)
unsigned int getHeight() const
Definition: vpImage.h:178
virtual void getCameraParameters(vpCameraParameters &camera) const
Definition: vpMbTracker.h:215
static void read(vpImage< unsigned char > &I, const std::string &filename)
Definition: vpImageIo.cpp:207
static void displayFrame(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, double size, const vpColor &color=vpColor::none, unsigned int thickness=1, const vpImagePoint &offset=vpImagePoint(0, 0))
unsigned int getDownScalingFactor()
Definition: vpDisplay.h:229
void setThreshold(const double &t)
Definition: vpMe.h:300
void display(const vpImage< unsigned char > &I, const vpHomogeneousMatrix &cMo, const vpCameraParameters &cam, const vpColor &col, const unsigned int thickness=1, const bool displayFullModel=false)
Class that defines a 2D point in an image. This class is useful for image processing and stores only ...
Definition: vpImagePoint.h:88
static void displayLine(const vpImage< unsigned char > &I, const vpImagePoint &ip1, const vpImagePoint &ip2, const vpColor &color, unsigned int thickness=1)
void setRange(const unsigned int &r)
Definition: vpMe.h:271
virtual void setClipping(const unsigned int &flags)
void addPoint(const vpPoint &P)
Definition: vpPose.cpp:137
unsigned int getWidth() const
Definition: vpImage.h:229
virtual std::pair< std::vector< vpPolygon >, std::vector< std::vector< vpPoint > > > getPolygonFaces(const bool orderPolygons=true, const bool useVisibility=true, const bool clipPolygon=false)