Fawkes API  Fawkes Development Version
siftpp.cpp
1 
2 /***************************************************************************
3  * siftpp.cpp - siftpp based classifier
4  *
5  * Created: Sat Apr 12 10:15:23 2008
6  * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7  *
8  ****************************************************************************/
9 
10 /* This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version. A runtime exception applies to
14  * this software (see LICENSE.GPL_WRE file mentioned below for details).
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Library General Public License for more details.
20  *
21  * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22  */
23 
24 #include <fvclassifiers/siftpp.h>
25 
26 #include <iostream>
27 #include <vector>
28 
29 //#ifdef SIFTPP_TIMETRACKER
30 #include <utils/time/clock.h>
31 #include <utils/time/tracker.h>
32 //#endif
33 
34 #include <core/exception.h>
35 #include <core/exceptions/software.h>
36 #include <fvutils/color/colorspaces.h>
37 #include <fvutils/color/conversions.h>
38 #include <fvutils/readers/png.h>
39 //#include <fvutils/writers/pnm.h>
40 //#include <fvutils/writers/png.h>
41 
42 //using namespace fawkes;
43 using namespace fawkes;
44 
45 namespace firevision {
46 
47 /** @class SiftppClassifier <fvclassifiers/siftpp.h>
48  * SIFTPP classifier.
49  *
50  * This class provides a classifier that uses SIFTPP to detect objects in a given
51  * image by matching features. The objects are reported back as regions of interest.
52  * Each ROI contains an object. ROIs with 11x11 are matched features.
53  *
54  * This code uses siftpp from http://vision.ucla.edu/~vedaldi/code/siftpp/siftpp.html
55  * and is partly based on code from their package.
56  *
57  * @author Stefan Schiffer
58  */
59 
60 /** Constructor.
61  * @param object_file file that contains an image of the object to detect
62  * @param samplingStep Initial sampling step
63  * @param octaves Number of analysed octaves
64  * @param levels Number of levels per octave
65  * @param magnif Keypoint magnification (default = 3)
66  * @param noorient rotation invariance (0) or upright (1)
67  * @param unnormalized Normalization of features (default 0)
68  */
69 SiftppClassifier::SiftppClassifier(const char *object_file,
70  int samplingStep,
71  int octaves,
72  int levels,
73  float magnif,
74  int noorient,
75  int unnormalized)
76 : Classifier("SiftppClassifier")
77 {
78  // params for FastHessian
79  samplingStep_ = samplingStep;
80  octaves_ = octaves;
81  levels_ = levels;
82  // params for Descriptors
83  first_ = -1;
84  threshold_ = 0.04f / levels_ / 2.0f;
85  edgeThreshold_ = 10.0f;
86  magnif_ = magnif;
87  noorient_ = noorient;
88  unnormalized_ = unnormalized;
89 
90  // descriptor vector length
91  vlen_ = 128;
92 
93  //#ifdef SIFTPP_TIMETRACKER
94  tt_ = new TimeTracker();
95  loop_count_ = 0;
96  ttc_objconv_ = tt_->add_class("ObjectConvert");
97  ttc_objfeat_ = tt_->add_class("ObjectFeatures");
98  ttc_imgconv_ = tt_->add_class("ImageConvert");
99  ttc_imgfeat_ = tt_->add_class("ImageFeatures");
100  ttc_matchin_ = tt_->add_class("Matching");
101  ttc_roimerg_ = tt_->add_class("MergeROIs");
102  //#endif
103 
104  //#ifdef SIFTPP_TIMETRACKER
105  tt_->ping_start(ttc_objconv_);
106  //#endif
107 
108  PNGReader pngr(object_file);
109  unsigned char *buf = malloc_buffer(pngr.colorspace(), pngr.pixel_width(), pngr.pixel_height());
110  pngr.set_buffer(buf);
111  pngr.read();
112 
113  unsigned int lwidth = pngr.pixel_width();
114  unsigned int lheight = pngr.pixel_height();
115  VL::pixel_t *im_pt = new VL::pixel_t[lwidth * lheight];
116  VL::pixel_t *start = im_pt;
117  //VL::pixel_t* end = start + lwidth*lheight ;
118  for (unsigned int h = 0; h < lheight; ++h) {
119  for (unsigned int w = 0; w < lwidth; ++w) {
120  int i = (buf[h * lwidth + w]);
121  VL::pixel_t norm = VL::pixel_t(255);
122  *start++ = VL::pixel_t(i) / norm;
123  }
124  }
125  // make image
126  obj_img_ = new VL::PgmBuffer();
127  obj_img_->width = lwidth;
128  obj_img_->height = lheight;
129  obj_img_->data = im_pt;
130 
131  if (!obj_img_) {
132  throw Exception("Could not load object file");
133  }
134 
135  //#ifdef SIFTPP_TIMETRACKER
136  tt_->ping_end(ttc_objconv_);
137  //#endif
138 
139  // save object image for debugging
140  //
141 
142  //#ifdef SIFTPP_TIMETRACKER
143  tt_->ping_start(ttc_objfeat_);
144  //#endif
145 
146  // COMPUTE OBJECT FEATURES
147  obj_features_.clear();
148  //obj_features_.reserve(1000);
149  obj_num_features_ = 0;
150 
151  sigman_ = .5;
152  sigma0_ = 1.6 * powf(2.0f, 1.0f / levels_);
153 
154  std::cout << "SiftppClassifier(ctor): init scalespace" << std::endl;
155  // initialize scalespace
156  VL::Sift sift(obj_img_->data,
157  obj_img_->width,
158  obj_img_->height,
159  sigman_,
160  sigma0_,
161  octaves_,
162  levels_,
163  first_,
164  -1,
165  levels_ + 1);
166 
167  std::cout << "SiftppClassifier(ctor): detect object keypoints" << std::endl;
168  // Run SIFTPP detector
169  sift.detectKeypoints(threshold_, edgeThreshold_);
170  // Number of keypoints
171  obj_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
172  std::cout << "SiftppClassifier(ctor): computed '" << obj_num_features_ << "' object-keypoints"
173  << std::endl;
174 
175  // set descriptor options
176  sift.setNormalizeDescriptor(!unnormalized_);
177  sift.setMagnification(magnif_);
178 
179  std::cout << "SiftppClassifier(ctor): run detector, compute ori and des ..." << std::endl;
180  // Run detector, compute orientations and descriptors
181  for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
182  ++iter) {
183  //Feature * feat = new Feature();
184  Feature feat;
185 
186  //std::cout << "SiftppClassifier(ctor): saving keypoint" << std::endl;
187  feat.key = (*iter);
188 
189  // detect orientations
190  VL::float_t angles[4];
191  int nangles;
192  if (!noorient_) {
193  nangles = sift.computeKeypointOrientations(angles, *iter);
194  } else {
195  nangles = 1;
196  angles[0] = VL::float_t(0);
197  }
198  feat.number_of_desc = nangles;
199  feat.descs = new VL::float_t *[nangles];
200 
201  //std::cout << "SiftppClassifier(ctor): computing '" << nangles << "' descriptors" << std::endl;
202  // compute descriptors
203  for (int a = 0; a < nangles; ++a) {
204  // out << setprecision(2) << iter->x << ' ' << setprecision(2) << iter->y << ' '
205  // << setprecision(2) << iter->sigma << ' ' << setprecision(3) << angles[a] ;
206  // compute descriptor
207  feat.descs[a] = new VL::float_t[vlen_];
208  sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]);
209  } // next angle
210  //std::cout << "SiftppClassifier(ctor): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
211 
212  // save feature
213  obj_features_.push_back(feat);
214 
215  } // next keypoint
216 
217  obj_num_features_ = obj_features_.size();
218  if (!obj_num_features_ > 0) {
219  throw Exception("Could not compute object features");
220  }
221  std::cout << "SiftppClassifier(ctor): computed '" << obj_num_features_ << "' features from object"
222  << std::endl;
223 
224  //#ifdef SIFTPP_TIMETRACKER
225  tt_->ping_end(ttc_objfeat_);
226  //#endif
227 }
228 
229 /** Destructor. */
231 {
232  //
233  delete obj_img_;
234  obj_features_.clear();
235  //
236  //delete image_;
237  img_features_.clear();
238 }
239 
240 std::list<ROI> *
242 {
243  //#ifdef SIFTPP_TIMETRACKER
244  tt_->ping_start(0);
245  //#endif
246 
247  // list of ROIs to return
248  std::list<ROI> *rv = new std::list<ROI>();
249 
250  // for ROI calculation
251  int x_min = _width;
252  int y_min = _height;
253  int x_max = 0;
254  int y_max = 0;
255 
256  //#ifdef SIFTPP_TIMETRACKER
257  tt_->ping_start(ttc_imgconv_);
258  //#endif
259  std::cout << "SiftppClassifier(classify): copy imgdat to SIFTPP Image" << std::endl;
260 
261  VL::pixel_t *im_pt = new VL::pixel_t[_width * _height];
262  VL::pixel_t *start = im_pt;
263  for (unsigned int h = 0; h < _height; ++h) {
264  for (unsigned int w = 0; w < _width; ++w) {
265  int i = (_src[h * _width + w]);
266  VL::pixel_t norm = VL::pixel_t(255);
267  *start++ = VL::pixel_t(i) / norm;
268  }
269  }
270  // make image
271  image_ = new VL::PgmBuffer();
272  image_->width = _width;
273  image_->height = _height;
274  image_->data = im_pt;
275 
276  //#ifdef SIFTPP_TIMETRACKER
277  tt_->ping_end(ttc_imgconv_);
278  //#endif
279 
280  /// Write image to verify correct operation
281  // nothing yet
282 
283  //#ifdef SIFTPP_TIMETRACKER
284  tt_->ping_start(ttc_imgfeat_);
285  //#endif
286 
287  // COMPUTE IMAGE FEATURES
288  img_features_.clear();
289  img_num_features_ = 0;
290  //img_features_.reserve(1000);
291 
292  std::cout << "SiftppClassifier(classify): init scalespace" << std::endl;
293  // initialize scalespace
294  VL::Sift sift(image_->data,
295  image_->width,
296  image_->height,
297  sigman_,
298  sigma0_,
299  octaves_,
300  levels_,
301  first_,
302  -1,
303  levels_ + 1);
304 
305  std::cout << "SiftppClassifier(classify): detect image keypoints" << std::endl;
306  // Run SIFTPP detector
307  sift.detectKeypoints(threshold_, edgeThreshold_);
308 
309  // Number of keypoints
310  img_num_features_ = sift.keypointsEnd() - sift.keypointsBegin();
311  std::cout << "SiftppClassifier(classify): Extracted '" << img_num_features_ << "' image keypoints"
312  << std::endl;
313 
314  // set descriptor options
315  sift.setNormalizeDescriptor(!unnormalized_);
316  sift.setMagnification(magnif_);
317 
318  std::cout << "SiftppClassifier(classify): run detector, compute ori and des ..." << std::endl;
319  // Run detector, compute orientations and descriptors
320  for (VL::Sift::KeypointsConstIter iter = sift.keypointsBegin(); iter != sift.keypointsEnd();
321  ++iter) {
322  Feature feat; // = new Feature();
323 
324  //std::cout << "SiftppClassifier(classify): saving keypoint" << std::endl;
325  feat.key = (*iter);
326 
327  //std::cout << "SiftppClassifier(classify): detect orientations" << std::endl;
328  // detect orientations
329  VL::float_t angles[4];
330  int nangles;
331  if (!noorient_) {
332  nangles = sift.computeKeypointOrientations(angles, *iter);
333  } else {
334  nangles = 1;
335  angles[0] = VL::float_t(0);
336  }
337  feat.number_of_desc = nangles;
338  feat.descs = new VL::float_t *[nangles];
339 
340  //std::cout << "SiftppClassifier(classify): computing '" << nangles << "' descriptors" << std::endl;
341  // compute descriptors
342  for (int a = 0; a < nangles; ++a) {
343  // compute descriptor
344  feat.descs[a] = new VL::float_t[vlen_];
345  sift.computeKeypointDescriptor(feat.descs[a], *iter, angles[a]);
346  } // next angle
347  //std::cout << "SiftppClassifier(classify): computed '" << feat.number_of_desc << "' descriptors." << std::endl;
348 
349  // save feature
350  img_features_.push_back(feat);
351 
352  } // next keypoint
353 
354  // Number of feature
355  img_num_features_ = img_features_.size();
356 
357  //#ifdef SIFTPP_TIMETRACKER
358  tt_->ping_end(ttc_imgfeat_);
359  //#endif
360 
361  std::cout << "SiftppClassifier(classify): Extracted '" << img_num_features_ << "' image features"
362  << std::endl;
363 
364  //#ifdef SIFTPP_TIMETRACKER
365  tt_->ping_start(ttc_matchin_);
366  //#endif
367  std::cout << "SiftppClassifier(classify): matching ..." << std::endl;
368 
369  std::vector<int> matches(obj_features_.size());
370  int m = 0;
371  for (unsigned i = 0; i < obj_features_.size(); i++) {
372  int match = findMatch(obj_features_[i], img_features_);
373  matches[i] = match;
374  if (match != -1) {
375  std::cout << "SiftppClassifier(classify): Matched feature " << i
376  << " in object image with feature " << match << " in image." << std::endl;
377  /// adding feature-ROI
378  ROI r((int)(img_features_[matches[i]].key.x) - 5,
379  (int)(img_features_[matches[i]].key.y) - 5,
380  11,
381  11,
382  _width,
383  _height);
384  rv->push_back(r);
385  // increment feature-match-count
386  ++m;
387  }
388  }
389 
390  //#ifdef SIFTPP_TIMETRACKER
391  tt_->ping_end(ttc_matchin_);
392  //#endif
393  std::cout << "SiftppClassifier(classify) matched '" << m << "' of '" << obj_features_.size()
394  << "' features in scene." << std::endl;
395 
396  std::cout << "SiftppClassifier(classify): computing ROI" << std::endl;
397  //#ifdef SIFTPP_TIMETRACKER
398  tt_->ping_start(ttc_roimerg_);
399  //#endif
400 
401  for (unsigned i = 0; i < matches.size(); i++) {
402  if (matches[i] != -1) {
403  if ((int)img_features_[matches[i]].key.x < x_min)
404  x_min = (int)img_features_[matches[i]].key.x;
405  if ((int)img_features_[matches[i]].key.y < y_min)
406  y_min = (int)img_features_[matches[i]].key.y;
407  if ((int)img_features_[matches[i]].key.x > x_max)
408  x_max = (int)img_features_[matches[i]].key.x;
409  if ((int)img_features_[matches[i]].key.y > y_max)
410  y_max = (int)img_features_[matches[i]].key.y;
411  }
412  }
413  if (m != 0) {
414  ROI r(x_min, y_min, x_max - x_min, y_max - y_min, _width, _height);
415  rv->push_back(r);
416  }
417 
418  //#ifdef SIFTPP_TIMETRACKER
419  tt_->ping_end(ttc_roimerg_);
420  //#endif
421 
422  //#ifdef SIFTPP_TIMETRACKER
423  tt_->ping_end(0);
424  //#endif
425 
426  //#ifdef SIFTPP_TIMETRACKER
427  // print timetracker statistics
428  tt_->print_to_stdout();
429  //#endif
430 
431  delete image_;
432 
433  std::cout << "SiftppClassifier(classify): done ... returning '" << rv->size() << "' ROIs."
434  << std::endl;
435  return rv;
436 }
437 
438 int
439 SiftppClassifier::findMatch(const Feature &ip1, const std::vector<Feature> &ipts)
440 {
441  double mind = 1e100, second = 1e100;
442  int match = -1;
443 
444  for (unsigned i = 0; i < ipts.size(); i++) {
445  if (ipts[i].number_of_desc != ip1.number_of_desc)
446  continue;
447  //std::cout << "SiftppClassifier(findMatch): number_of_desc matched!" << std::endl;
448  for (int j = 0; j < ip1.number_of_desc; ++j) {
449  double d = distSquare(ipts[i].descs[j], ip1.descs[j], vlen_);
450 
451  if (d < mind) {
452  second = mind;
453  mind = d;
454  match = i;
455  } else if (d < second) {
456  second = d;
457  }
458  }
459  }
460 
461  if (mind < 0.5 * second)
462  return match;
463 
464  return -1;
465 }
466 
467 double
468 SiftppClassifier::distSquare(VL::float_t *v1, VL::float_t *v2, int n)
469 {
470  double dsq = 0.;
471  while (n--) {
472  dsq += (v1[n - 1] - v2[n - 1]) * (v1[n - 1] - v2[n - 1]);
473  }
474  //std::cout << " dsq: '" << dsq << "'" << std::endl;
475  return dsq;
476 }
477 
478 } // end namespace firevision
firevision::Classifier
Definition: classifier.h:39
firevision::PNGReader::colorspace
virtual colorspace_t colorspace()
Definition: png.cpp:183
firevision::SiftppClassifier::~SiftppClassifier
virtual ~SiftppClassifier()
Destructor.
Definition: siftpp.cpp:229
fawkes::TimeTracker::ping_start
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:217
firevision::SiftppClassifier::classify
virtual std::list< ROI > * classify()
Definition: siftpp.cpp:240
firevision::Classifier::_height
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:57
fawkes::TimeTracker::print_to_stdout
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:306
firevision::PNGReader::pixel_height
virtual unsigned int pixel_height()
Definition: png.cpp:199
firevision::ROI
Definition: roi.h:58
firevision::SiftppClassifier::Feature::key
VL::Sift::Keypoint key
keypoint
Definition: siftpp.h:64
firevision::PNGReader::read
virtual void read()
Definition: png.cpp:209
firevision::Classifier::_width
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:55
firevision::SiftppClassifier::Feature
Siftpp Feature struct.
Definition: siftpp.h:62
firevision::PNGReader
Definition: png.h:37
firevision::PNGReader::set_buffer
virtual void set_buffer(unsigned char *yuv422planar_buffer)
Definition: png.cpp:177
fawkes::TimeTracker::add_class
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:148
fawkes
firevision::SiftppClassifier::Feature::descs
VL::float_t ** descs
descriptors
Definition: siftpp.h:66
fawkes::TimeTracker
Definition: tracker.h:40
firevision::PNGReader::pixel_width
virtual unsigned int pixel_width()
Definition: png.cpp:189
firevision::Classifier::_src
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:53
fawkes::TimeTracker::ping_end
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:242
firevision::SiftppClassifier::Feature::number_of_desc
int number_of_desc
number of descriptors
Definition: siftpp.h:65
fawkes::Exception
Definition: exception.h:39