Python eval code, finally! (Thanks to @bharath272)

Author: rbgirshick

README.md

@@ -59,7 +59,7 @@ If you find Faster R-CNN useful in your research, please consider citing:
 
   You can download my [Makefile.config](http://www.cs.berkeley.edu/~rbg/fast-rcnn-data/Makefile.config) for reference.
 2. Python packages you might not have: `cython`, `python-opencv`, `easydict`
-3. [optional] MATLAB (required for PASCAL VOC evaluation only)
+3. [Optional] MATLAB is required for **official** PASCAL VOC evaluation only. The code now includes unofficial Python evaluation code.
 
 ### Requirements: hardware
 

lib/datasets/VOCdevkit-matlab-wrapper/voc_eval.m

@@ -1,11 +1,11 @@
-function res = voc_eval(path, comp_id, test_set, output_dir, rm_res)
+function res = voc_eval(path, comp_id, test_set, output_dir)
 
 VOCopts = get_voc_opts(path);
 VOCopts.testset = test_set;
 
 for i = 1:length(VOCopts.classes)
   cls = VOCopts.classes{i};
-  res(i) = voc_eval_cls(cls, VOCopts, comp_id, output_dir, rm_res);
+  res(i) = voc_eval_cls(cls, VOCopts, comp_id, output_dir);
 end
 
 fprintf('\n~~~~~~~~~~~~~~~~~~~~\n');
@@ -15,7 +15,7 @@
 fprintf('%.1f\n', mean(aps) * 100);
 fprintf('~~~~~~~~~~~~~~~~~~~~\n');
 
-function res = voc_eval_cls(cls, VOCopts, comp_id, output_dir, rm_res)
+function res = voc_eval_cls(cls, VOCopts, comp_id, output_dir)
 
 test_set = VOCopts.testset;
 year = VOCopts.dataset(4:end);
@@ -53,8 +53,4 @@
 save([output_dir '/' cls '_pr.mat'], ...
      'res', 'recall', 'prec', 'ap', 'ap_auc');
 
-if rm_res
-  delete(res_fn);
-end
-
 rmpath(fullfile(VOCopts.datadir, 'VOCcode'));

lib/datasets/pascal_voc.py

@@ -7,13 +7,15 @@
 
 import os
 from datasets.imdb import imdb
-import xml.dom.minidom as minidom
+import xml.etree.ElementTree as ET
 import numpy as np
 import scipy.sparse
 import scipy.io as sio
 import utils.cython_bbox
 import cPickle
 import subprocess
+import uuid
+from voc_eval import voc_eval
 from fast_rcnn.config import cfg
 
 class pascal_voc(imdb):
@@ -35,13 +37,16 @@ def __init__(self, image_set, year, devkit_path=None):
         self._image_index = self._load_image_set_index()
         # Default to roidb handler
         self._roidb_handler = self.selective_search_roidb
+        self._salt = str(uuid.uuid4())
+        self._comp_id = 'comp4'
 
         # PASCAL specific config options
-        self.config = {'cleanup'  : True,
-                       'use_salt' : True,
-                       'top_k'    : 2000,
-                       'use_diff' : False,
-                       'rpn_file' : None}
+        self.config = {'cleanup'     : True,
+                       'use_salt'    : True,
+                       'top_k'       : 2000,
+                       'use_diff'    : False,
+                       'matlab_eval' : False,
+                       'rpn_file'    : None}
 
         assert os.path.exists(self._devkit_path), \
                 'VOCdevkit path does not exist: {}'.format(self._devkit_path)
@@ -172,21 +177,15 @@ def _load_pascal_annotation(self, index):
         format.
         """
         filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
-        # print 'Loading: {}'.format(filename)
-        def get_data_from_tag(node, tag):
-            return node.getElementsByTagName(tag)[0].childNodes[0].data
-
-        with open(filename) as f:
-            data = minidom.parseString(f.read())
-
-        objs = data.getElementsByTagName('object')
+        tree = ET.parse(filename)
+        objs = tree.findall('object')
         if not self.config['use_diff']:
             # Exclude the samples labeled as difficult
-            non_diff_objs = [obj for obj in objs
-                             if int(get_data_from_tag(obj, 'difficult')) == 0]
+            non_diff_objs = [
+                obj for obj in objs if int(obj.find('difficult').text) == 0]
             if len(non_diff_objs) != len(objs):
-                print 'Removed {} difficult objects' \
-                    .format(len(objs) - len(non_diff_objs))
+                print 'Removed {} difficult objects'.format(
+                    len(objs) - len(non_diff_objs))
             objs = non_diff_objs
         num_objs = len(objs)
 
@@ -196,13 +195,13 @@ def get_data_from_tag(node, tag):
 
         # Load object bounding boxes into a data frame.
         for ix, obj in enumerate(objs):
+            bbox = obj.find('bndbox')
             # Make pixel indexes 0-based
-            x1 = float(get_data_from_tag(obj, 'xmin')) - 1
-            y1 = float(get_data_from_tag(obj, 'ymin')) - 1
-            x2 = float(get_data_from_tag(obj, 'xmax')) - 1
-            y2 = float(get_data_from_tag(obj, 'ymax')) - 1
-            cls = self._class_to_ind[
-                    str(get_data_from_tag(obj, "name")).lower().strip()]
+            x1 = float(bbox.find('xmin').text) - 1
+            y1 = float(bbox.find('ymin').text) - 1
+            x2 = float(bbox.find('xmax').text) - 1
+            y2 = float(bbox.find('ymax').text) - 1
+            cls = self._class_to_ind[obj.find('name').text.lower().strip()]
             boxes[ix, :] = [x1, y1, x2, y2]
             gt_classes[ix] = cls
             overlaps[ix, cls] = 1.0
@@ -214,20 +213,28 @@ def get_data_from_tag(node, tag):
                 'gt_overlaps' : overlaps,
                 'flipped' : False}
 
+    def _get_comp_id(self):
+        comp_id = (self._comp_id + '_' + self._salt if self.config['use_salt']
+            else self._comp_id)
+        return comp_id
+
+    def _get_voc_results_file_template(self):
+        # VOCdevkit/results/VOC2007/Main/<comp_id>_det_test_aeroplane.txt
+        filename = self._get_comp_id() + '_det_' + self._image_set + '_{:s}.txt'
+        path = os.path.join(
+            self._devkit_path,
+            'results',
+            'VOC' + self._year,
+            'Main',
+            filename)
+        return path
+
     def _write_voc_results_file(self, all_boxes):
-        use_salt = self.config['use_salt']
-        comp_id = 'comp4'
-        if use_salt:
-            comp_id += '-{}'.format(os.getpid())
-
-        # VOCdevkit/results/VOC2007/Main/comp4-44503_det_test_aeroplane.txt
-        path = os.path.join(self._devkit_path, 'results', 'VOC' + self._year,
-                            'Main', comp_id + '_')
         for cls_ind, cls in enumerate(self.classes):
             if cls == '__background__':
                 continue
             print 'Writing {} VOC results file'.format(cls)
-            filename = path + 'det_' + self._image_set + '_' + cls + '.txt'
+            filename = self._get_voc_results_file_template().format(cls)
             with open(filename, 'wt') as f:
                 for im_ind, index in enumerate(self.image_index):
                     dets = all_boxes[cls_ind][im_ind]
@@ -239,25 +246,76 @@ def _write_voc_results_file(self, all_boxes):
                                 format(index, dets[k, -1],
                                        dets[k, 0] + 1, dets[k, 1] + 1,
                                        dets[k, 2] + 1, dets[k, 3] + 1))
-        return comp_id
-
-    def _do_matlab_eval(self, comp_id, output_dir='output'):
-        rm_results = self.config['cleanup']
 
+    def _do_python_eval(self, output_dir = 'output'):
+        print '--------------------------------------------------------------'
+        print 'Computing results with **unofficial** Python eval code.'
+        print 'Results should be very close to the official MATLAB eval code.'
+        print 'Recompute with `./tools/reval.py --matlab ...` for your paper.'
+        print '--------------------------------------------------------------'
+        annopath = os.path.join(
+            self._devkit_path,
+            'VOC' + self._year,
+            'Annotations',
+            '{:s}.xml')
+        imagesetfile = os.path.join(
+            self._devkit_path,
+            'VOC' + self._year,
+            'ImageSets',
+            'Main',
+            self._image_set + '.txt')
+        cachedir = os.path.join(self._devkit_path, 'annotations_cache')
+        aps = []
+        # The PASCAL VOC metric changed in 2010
+        use_07_metric = True if int(self._year) < 2010 else False
+        print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No')
+        if not os.path.isdir(output_dir):
+            os.mkdir(output_dir)
+        for i, cls in enumerate(self._classes):
+            if cls == '__background__':
+                continue
+            filename = self._get_voc_results_file_template().format(cls)
+            rec, prec, ap = voc_eval(
+                filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5,
+                use_07_metric=use_07_metric)
+            aps += [ap]
+            print('AP for {} = {:.4f}'.format(cls, ap))
+            with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f:
+                cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
+        print('Mean AP = {:.4f}'.format(np.mean(aps)))
+        print('~~~~~~~~')
+        print('Results:')
+        for ap in aps:
+            print('{:.3f}'.format(ap))
+        print('{:.3f}'.format(np.mean(aps)))
+        print('~~~~~~~~')
+
+    def _do_matlab_eval(self, output_dir='output'):
+        print '-----------------------------------------------------'
+        print 'Computing results with the official MATLAB eval code.'
+        print '-----------------------------------------------------'
         path = os.path.join(cfg.ROOT_DIR, 'lib', 'datasets',
                             'VOCdevkit-matlab-wrapper')
         cmd = 'cd {} && '.format(path)
         cmd += '{:s} -nodisplay -nodesktop '.format(cfg.MATLAB)
         cmd += '-r "dbstop if error; '
-        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
-               .format(self._devkit_path, comp_id,
-                       self._image_set, output_dir, int(rm_results))
+        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\'); quit;"' \
+               .format(self._devkit_path, self._get_comp_id(),
+                       self._image_set, output_dir)
         print('Running:\n{}'.format(cmd))
         status = subprocess.call(cmd, shell=True)
 
     def evaluate_detections(self, all_boxes, output_dir):
-        comp_id = self._write_voc_results_file(all_boxes)
-        self._do_matlab_eval(comp_id, output_dir)
+        self._write_voc_results_file(all_boxes)
+        self._do_python_eval(output_dir)
+        if self.config['matlab_eval']:
+            self._do_matlab_eval(output_dir)
+        if self.config['cleanup']:
+            for cls in self._classes:
+                if cls == '__background__':
+                    continue
+                filename = self._get_voc_results_file_template().format(cls)
+                os.remove(filename)
 
     def competition_mode(self, on):
         if on: