Load cognata validation set from csv file

-includes script to generate data splits

Author: rod409

automotive/2d-object-detection/accuracy_cognata.py

@@ -9,10 +9,11 @@
 import pprint
 import numpy as np
 from cognata import collate_fn, Cognata, prepare_cognata, train_val_split
+import cognata_labels
 from transform import SSDTransformer
 import importlib
 import torch
-from utils import generate_dboxes, Encoder
+from utils import generate_dboxes, read_dataset_csv
 from torchmetrics.detection.mean_ap import MeanAveragePrecision
 # pylint: disable=missing-docstring
 
@@ -25,7 +26,7 @@ def get_args():
         required=True,
         help="path to mlperf_log_accuracy.json")
     parser.add_argument(
-        "--cognata-dir",
+        "--dataset-path",
         required=True,
         help="cognata dataset directory")
     parser.add_argument(
@@ -51,25 +52,26 @@ def main():
     with open(args.mlperf_accuracy_file, "r") as f:
         results = json.load(f)
 
-    detections = {}
-    image_ids = set()
     seen = set()
-    no_results = 0
     config = importlib.import_module('config.' + args.config)
     folders = config.dataset['folders']
     cameras = config.dataset['cameras']
     ignore_classes = [2, 25, 31]
     if 'ignore_classes' in config.dataset:
         ignore_classes = config.dataset['ignore_classes']
-    files, label_map, label_info = prepare_cognata(
-        args.cognata_dir, folders, cameras, ignore_classes)
-    files = train_val_split(files)
+    if config.dataset['use_label_file']:
+        label_map = cognata_labels.label_map
+        label_info = cognata_labels.label_info
+    else:
+        _, label_map, label_info = prepare_cognata(args.dataset_path, folders, cameras)
+    files = read_dataset_csv("val_set.csv")
+    files = [{'img': os.path.join(args.dataset_path, f['img']), 'ann': os.path.join(args.dataset_path, f['ann'])} for f in files]
     dboxes = generate_dboxes(config.model, model="ssd")
     image_size = config.model['image_size']
     val_set = Cognata(
         label_map,
         label_info,
-        files['val'],
+        files,
         ignore_classes,
         SSDTransformer(
             dboxes,

automotive/2d-object-detection/cognata.py

@@ -181,10 +181,11 @@ def prepare_cognata(root, folders, cameras, ignore_classes=[2, 25, 31]):
     return files, label_map, label_info
 
 
-def train_val_split(files):
+def train_val_split(files, calibration_length=200):
     random.Random(5).shuffle(files)
     val_index = round(len(files) * 0.8)
-    return {'train': files[:val_index], 'val': files[val_index:]}
+    calibration_index = len(files) - calibration_length
+    return {'train': files[:val_index], 'val': files[val_index:calibration_index], 'calibration': files[calibration_index:]}
 
 
 class PostProcessCognata:

automotive/2d-object-detection/cognata_data_split.py

@@ -0,0 +1,44 @@
+import argparse
+import importlib
+from cognata import Cognata, prepare_cognata, train_val_split
+import csv
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", help="config file")
+    parser.add_argument(
+        "--dataset-path",
+        required=True,
+        help="path to the dataset")
+    parser.add_argument(
+        "--calibration-size",
+        default=200,
+        type=int,
+        help="number of items in calibration dataset",
+    )
+    args = parser.parse_args()
+    return args
+
+def write_data_to_csv(file_path, files):
+    with open(file_path, mode='w', newline='') as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(['img', 'ann'])
+        for file in files:
+            writer.writerow([file['img'], file['ann']])
+
+def main():
+    args = get_args()
+    config = importlib.import_module('config.' + args.config)
+    folders = config.dataset['folders']
+    cameras = config.dataset['cameras']
+    files, label_map, label_info = prepare_cognata(
+        args.dataset_path, folders, cameras)
+    files = train_val_split(files, args.calibration_size)
+    for file_set in files.values():
+        for file in file_set:
+            file['img'] = file['img'].replace(args.dataset_path, '')
+            file['ann'] = file['ann'].replace(args.dataset_path, '')
+    write_data_to_csv('val_set.csv', files['val'])
+    write_data_to_csv('calibration_set.csv', files['calibration'])
+
+main()

automotive/2d-object-detection/config/baseline_8MP_ss_scales_fm1_5x5_all.py

@@ -41,17 +41,13 @@
 )
 
 dataset = dict(
-    folders=['10001_Urban_Clear_Noon'],
-    cameras=[
-        'Cognata_Camera_01_8M',
-        'Cognata_Camera_02_8M',
-        'Cognata_Camera_03_8M'],
-    use_label_file=True
-)
-
-'''folders = ['10001_Urban_Clear_Noon', '10002_Urban_Clear_Morning', '10003_Urban_Clear_Noon', '10004_Urban_Clear_Noon', '10005_Urban_Clear_Noon',
+    folders = ['10001_Urban_Clear_Noon', '10002_Urban_Clear_Morning', '10003_Urban_Clear_Noon', '10004_Urban_Clear_Noon', '10005_Urban_Clear_Noon',
                '10006_Urban_Clear_Noon', '10007_Highway_Clear_Morning', '10008_Highway_Clear_Noon', '10009_Urban_Rain_Morning', '10010_Urban_Rain_Morning', '10011_Urban_Rain_Evening',
                '10012_Urban_Rain_Evening', '10013_Highway_Rain_Morning', '10014_Highway_Rain_Noon', '10015_Highway_Clear_Evening', '10016_Highway_Clear_Evening',
                '10017_Urban_Rain_Evening', '10018_Urban_Rain_Evening', '10019_Urban_HeavyRain_Noon', '10020_Urban_HeavyRain_Afternoon', '10021_Urban_Clear_Morning', '10022_Urban_Clear_Morning',
                '10023_Urban_Rain_Morning', '10024_Urban_Rain_Evening', '10025_Urban_HeavyRain_Evening', '10026_Urban_HeavyRain_Afternoon'],
-    cameras = ['Cognata_Camera_01_8M', 'Cognata_Camera_02_8M', 'Cognata_Camera_03_8M']'''
+    cameras = ['Cognata_Camera_01_8M', 'Cognata_Camera_02_8M', 'Cognata_Camera_03_8M'],
+    use_label_file=True
+)
+
+

automotive/2d-object-detection/main.py

@@ -20,12 +20,12 @@
 import mlperf_loadgen as lg
 import numpy as np
 import torch
-
+import csv
 import dataset
 import cognata
 from transform import SSDTransformer
 import importlib
-from utils import generate_dboxes, Encoder
+from utils import generate_dboxes, Encoder, read_dataset_csv
 from cognata import Cognata, prepare_cognata, train_val_split
 import cognata_labels
 
@@ -300,7 +300,6 @@ def finish(self):
         for worker in self.workers:
             worker.join()
 
-
 def main():
     args = get_args()
 
@@ -311,12 +310,15 @@ def main():
     transformer = SSDTransformer(dboxes, image_size, val=True)
     folders = config.dataset['folders']
     cameras = config.dataset['cameras']
-    files, label_map, label_info = prepare_cognata(
-        args.dataset_path, folders, cameras)
-    files = train_val_split(files)
+
     if config.dataset['use_label_file']:
         label_map = cognata_labels.label_map
         label_info = cognata_labels.label_info
+    else:
+        _, label_map, label_info = prepare_cognata(args.dataset_path, folders, cameras)
+    
+    files = read_dataset_csv("val_set.csv")
+    files = [{'img': os.path.join(args.dataset_path, f['img']), 'ann': os.path.join(args.dataset_path, f['ann'])} for f in files]
     # find backend
     backend = get_backend(
         # TODO: pass model, inference and backend arguments
@@ -349,7 +351,7 @@ def main():
     ds = dataset_class(
         label_map=label_map,
         label_info=label_info,
-        files=files['val'],
+        files=files,
         ignore_classes=[2, 25, 31],
         transform=transformer)
 

automotive/2d-object-detection/utils.py

@@ -27,7 +27,7 @@
 import numpy as np
 import itertools
 from math import sqrt
-
+import csv
 import torch
 import torch.nn.functional as F
 from torchvision.ops.boxes import box_iou, box_convert
@@ -218,3 +218,11 @@ def generate_dboxes(config, model="ssd"):
         aspect_ratios = [[2,3], [2, 3], [2, 3], [2, 3], [2,3], [2,3]]
         dboxes = DefaultBoxes(figsize, feat_size, steps, scales, aspect_ratios)
     return dboxes
+
+def read_dataset_csv(file_path):
+    files = []
+    with open(file_path, mode='r') as csv_file:
+        csv_reader = csv.DictReader(csv_file)
+        for row in csv_reader:
+            files.append(row)
+    return files

automotive/semantic-segmentation/accuracy_cognata.py

@@ -5,11 +5,12 @@
 """
 import argparse
 import json
+import os
 import numpy as np
 from cognata import Cognata, prepare_cognata, train_val_split
 import torch
 import cognata_scenarios
-from utils import StreamSegMetrics
+from utils import StreamSegMetrics, read_dataset_csv
 from utils import ext_transforms as et
 # pylint: disable=missing-docstring
 
@@ -22,7 +23,7 @@ def get_args():
         required=True,
         help="path to mlperf_log_accuracy.json")
     parser.add_argument(
-        "--cognata-dir",
+        "--dataset-path",
         required=True,
         help="cognata dataset directory")
     parser.add_argument(
@@ -55,25 +56,20 @@ def main():
     with open(args.mlperf_accuracy_file, "r") as f:
         results = json.load(f)
 
-    detections = {}
-    image_ids = set()
     seen = set()
-    no_results = 0
     num_classes = 19
     metrics = StreamSegMetrics(num_classes)
     metrics.reset()
-    files = prepare_cognata(args.cognata_dir, cognata_scenarios.folders, cognata_scenarios.cameras)
-    split = train_val_split(files)
+    files = read_dataset_csv("val_set.csv")
+    files = [{'img': os.path.join(args.dataset_path, f['img']), 'label': os.path.join(args.dataset_path, f['label'])} for f in files]
     image_size = args.image_size
     val_transform = et.ExtCompose([
         et.ExtResize((image_size[0], image_size[1])),
         et.ExtToTensor(),
         et.ExtNormalize(mean=[0.485, 0.456, 0.406],
                         std=[0.229, 0.224, 0.225]),
     ])
-    val_loader = Cognata(split['val'], transform=val_transform)
-    preds = []
-    targets = []
+    val_loader = Cognata(files, transform=val_transform)
     for j in results:
         idx = j['qsl_idx']
         # de-dupe in case loadgen sends the same image multiple times

automotive/semantic-segmentation/cognata.py

@@ -24,10 +24,11 @@ def prepare_cognata(root, folders, cameras):
 
     return files
 
-def train_val_split(files):
+def train_val_split(files, calibration_length=200):
     random.Random(5).shuffle(files)
-    val_index = round(len(files)*0.8)
-    return {'train': files[:val_index], 'val': files[val_index:]}
+    val_index = round(len(files) * 0.8)
+    calibration_index = len(files) - calibration_length
+    return {'train': files[:val_index], 'val': files[val_index:calibration_index], 'calibration': files[calibration_index:]}
 
 class Cognata(dataset.Dataset):
     CognataClass = namedtuple('CognataClass', ['name', 'id', 'train_id', 'color'])

automotive/semantic-segmentation/cognata_data_split.py

@@ -0,0 +1,41 @@
+import argparse
+import importlib
+from cognata import Cognata, prepare_cognata, train_val_split
+import cognata_scenarios
+import csv
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset-path",
+        required=True,
+        help="path to the dataset")
+    parser.add_argument(
+        "--calibration-size",
+        default=200,
+        type=int,
+        help="number of items in calibration dataset",
+    )
+    args = parser.parse_args()
+    return args
+
+def write_data_to_csv(file_path, files):
+    with open(file_path, mode='w', newline='') as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(['img', 'label'])
+        for file in files:
+            writer.writerow([file['img'], file['label']])
+
+def main():
+    args = get_args()
+    files = prepare_cognata(
+        args.dataset_path, cognata_scenarios.folders, cognata_scenarios.cameras)
+    files = train_val_split(files, args.calibration_size)
+    for file_set in files.values():
+        for file in file_set:
+            file['img'] = file['img'].replace(args.dataset_path, '')
+            file['label'] = file['label'].replace(args.dataset_path, '')
+    write_data_to_csv('val_set.csv', files['val'])
+    write_data_to_csv('calibration_set.csv', files['calibration'])
+
+main()

automotive/semantic-segmentation/cognata_scenarios.py

@@ -1,9 +1,7 @@
 
-'''folders = ['10001_Urban_Clear_Noon', '10002_Urban_Clear_Morning', '10003_Urban_Clear_Noon', '10004_Urban_Clear_Noon', '10005_Urban_Clear_Noon',
+folders = ['10001_Urban_Clear_Noon', '10002_Urban_Clear_Morning', '10003_Urban_Clear_Noon', '10004_Urban_Clear_Noon', '10005_Urban_Clear_Noon',
                '10006_Urban_Clear_Noon', '10007_Highway_Clear_Morning', '10008_Highway_Clear_Noon', '10009_Urban_Rain_Morning', '10010_Urban_Rain_Morning', '10011_Urban_Rain_Evening',
                '10012_Urban_Rain_Evening', '10013_Highway_Rain_Morning', '10014_Highway_Rain_Noon', '10015_Highway_Clear_Evening', '10016_Highway_Clear_Evening',
                '10017_Urban_Rain_Evening', '10018_Urban_Rain_Evening', '10019_Urban_HeavyRain_Noon', '10020_Urban_HeavyRain_Afternoon', '10021_Urban_Clear_Morning', '10022_Urban_Clear_Morning',
                '10023_Urban_Rain_Morning', '10024_Urban_Rain_Evening', '10025_Urban_HeavyRain_Evening', '10026_Urban_HeavyRain_Afternoon']
-cameras = ['Cognata_Camera_01_8M', 'Cognata_Camera_02_8M', 'Cognata_Camera_03_8M']'''
-folders = ['10001_Urban_Clear_Noon']
 cameras = ['Cognata_Camera_01_8M', 'Cognata_Camera_02_8M', 'Cognata_Camera_03_8M']