Add differential results of regression benchmark train results.

.dev/gather_benchmark_train_results.py

@@ -10,13 +10,11 @@
 def parse_args():
     parser = argparse.ArgumentParser(
         description='Gather benchmarked models train results')
-    parser.add_argument(
-        'txt_path', type=str, help='txt path output by benchmark_filter')
+    parser.add_argument('config', help='test config file path')
     parser.add_argument(
         'root',
         type=str,
         help='root path of benchmarked models to be gathered')
-    parser.add_argument('config', help='test config file path')
     parser.add_argument(
         '--out',
         type=str,
@@ -33,18 +31,19 @@ def parse_args():
     root_path = args.root
     metrics_out = args.out
 
-    cfg = Config.fromfile(args.config)
+    evaluation_cfg = Config.fromfile(args.config)
 
     result_dict = {}
-    with open(args.txt_path, 'r') as f:
-        model_cfgs = f.readlines()
-        for _, config in enumerate(model_cfgs):
-            config = config.strip()
+    for model_key in evaluation_cfg:
+        model_infos = evaluation_cfg[model_key]
+        if not isinstance(model_infos, list):
+            model_infos = [model_infos]
+        for model_info in model_infos:
+            config = model_info['config']
 
             # benchmark train dir
             model_name = osp.split(osp.dirname(config))[1]
             config_name = osp.splitext(osp.basename(config))[0]
-            model_info = cfg[model_name][config_name]
             exp_dir = osp.join(root_path, model_name, config_name)
             if not osp.exists(exp_dir):
                 print(f'{config} hasn\'t {exp_dir}')
@@ -74,27 +73,28 @@ def parse_args():
                 continue
 
             differential_results = dict()
-            old_performance = dict()
-            for performance in model_performance:
-                if performance in ['mIoU']:
-                    metric = round(model_performance[performance] * 100, 1)
-                    old_metric = model_info[performance]
-                    old_performance[performance] = old_metric
-                    model_performance[performance] = metric
+            old_results = dict()
+            new_results = dict()
+            for metric_key in model_performance:
+                if metric_key in ['mIoU']:
+                    metric = round(model_performance[metric_key] * 100, 1)
+                    old_metric = model_info['metric'][metric_key]
+                    old_results[metric_key] = old_metric
+                    new_results[metric_key] = metric
                     differential = metric - old_metric
                     flag = '+' if differential > 0 else '-'
                     differential_results[
-                        performance] = f'{flag}{abs(differential):.2f}'
+                        metric_key] = f'{flag}{abs(differential):.2f}'
             result_dict[config] = dict(
                 differential_results=differential_results,
-                old_results=old_performance,
-                new_results=model_performance,
+                old_results=old_results,
+                new_results=new_results,
             )
 
-        # 4 save or print results
-        if metrics_out:
-            mmcv.dump(result_dict, metrics_out, indent=4)
-        print('===================================')
-        for config_name, metrics in result_dict.items():
-            print(config_name, metrics)
-        print('===================================')
+    # 4 save or print results
+    if metrics_out:
+        mmcv.dump(result_dict, metrics_out, indent=4)
+    print('===================================')
+    for config_name, metrics in result_dict.items():
+        print(config_name, metrics)
+    print('===================================')