Added mAP_lim.py which is mAP but limited to a max. k of 1023

Author: Confusezius

metrics/__init__.py

@@ -1,5 +1,5 @@
 from metrics import e_recall, dists, rho_spectrum
-from metrics import nmi, f1, mAP, mAP_c, mAP_1000
+from metrics import nmi, f1, mAP, mAP_c, mAP_1000, mAP_lim
 import numpy as np
 import faiss
 import torch
@@ -17,6 +17,8 @@ def select(metricname, opt):
         return mAP.Metric()
     elif metricname=='mAP_c':
         return mAP_c.Metric()
+    elif metricname=='mAP_lim':
+        return mAP_lim.Metric()
     elif metricname=='mAP_1000':
         return mAP_1000.Metric()
     elif metricname=='f1':

metrics/mAP_lim.py

@@ -0,0 +1,34 @@
+import torch
+import numpy as np
+import faiss
+
+
+
+class Metric():
+    def __init__(self, **kwargs):
+        self.requires = ['features', 'target_labels']
+        self.name     = 'mAP'
+
+    def __call__(self, target_labels, features):
+        labels, freqs = np.unique(target_labels, return_counts=True)
+        ## Account for faiss-limit at k=1023
+        R             = min(1023,len(features))
+
+        faiss_search_index  = faiss.IndexFlatL2(features.shape[-1])
+        faiss_search_index.add(features)
+        nearest_neighbours  = faiss_search_index.search(features, int(R+1))[1][:,1:]
+
+        target_labels = target_labels.reshape(-1)
+        nn_labels = target_labels[nearest_neighbours]
+
+        avg_r_precisions = []
+        for label, freq in zip(labels, freqs):
+            rows_with_label = np.where(target_labels==label)[0]
+            for row in rows_with_label:
+                n_recalled_samples           = np.arange(1,R+1)
+                target_label_occ_in_row      = nn_labels[row,:]==label
+                cumsum_target_label_freq_row = np.cumsum(target_label_occ_in_row)
+                avg_r_pr_row = np.sum(cumsum_target_label_freq_row*target_label_occ_in_row/n_recalled_samples)/freq
+                avg_r_precisions.append(avg_r_pr_row)
+
+        return np.mean(avg_r_precisions)

parameters.py

@@ -38,7 +38,7 @@ def basic_training_parameters(parser):
     ##### Evaluation Parameters
     parser.add_argument('--no_train_metrics', action='store_true',   help='Flag. If set, evaluation metrics are not computed for the training data. Saves a forward pass over the full training dataset.')
     parser.add_argument('--evaluate_on_gpu',  action='store_true',   help='Flag. If set, all metrics, when possible, are computed on the GPU (requires Faiss-GPU).')
-    parser.add_argument('--evaluation_metrics', nargs='+', default=['e_recall@1', 'e_recall@2', 'e_recall@4', 'nmi', 'f1', 'mAP_1000', 'mAP_c', 'dists@intra', 'dists@inter', 'dists@intra_over_inter', 'rho_spectrum@0', 'rho_spectrum@-1', 'rho_spectrum@1', 'rho_spectrum@2', 'rho_spectrum@10'], type=str, help='Metrics to evaluate performance by.')
+    parser.add_argument('--evaluation_metrics', nargs='+', default=['e_recall@1', 'e_recall@2', 'e_recall@4', 'nmi', 'f1', 'mAP_1000', 'mAP_lim', 'mAP_c', 'dists@intra', 'dists@inter', 'dists@intra_over_inter', 'rho_spectrum@0', 'rho_spectrum@-1', 'rho_spectrum@1', 'rho_spectrum@2', 'rho_spectrum@10'], type=str, help='Metrics to evaluate performance by.')
     parser.add_argument('--storage_metrics',    nargs='+', default=['e_recall@1'],     type=str, help='Improvement in these metrics on a dataset trigger checkpointing.')
     parser.add_argument('--evaltypes',          nargs='+', default=['discriminative'], type=str, help='The network may produce multiple embeddings (ModuleDict, relevant for e.g. DiVA). If the key is listed here, the entry will be evaluated on the evaluation metrics.\
                                                                                                        Note: One may use Combined_embed1_embed2_..._embedn-w1-w1-...-wn to compute evaluation metrics on weighted (normalized) combinations.')