Add v5.1 submission checker (mlcommons#2204)

pgmpablo157321 · github-actions[bot] · arjunsuresh · web-flow · commit 0910c6e30098 · 2025-06-18T00:09:28.000+01:00
* Add v5.1 submission checker

* [Automated Commit] Format Codebase

* Restore resnet for edge

* Update auto-update-dev.yml

* Update generate_final_report.py

* Update submission_checker.py

* Add whisper configuration to checker

* [Automated Commit] Format Codebase

* Update auto-update-dev.yml

* [Automated Commit] Format Codebase

* Update auto-update-dev.yml

* Only require one of server or interactive

* Add v5.1 seeds

* Fix typo, add commas

---------

Co-authored-by: github-actions[bot] &lt;github-actions[bot]@users.noreply.github.com&gt;
Co-authored-by: Arjun Suresh &lt;arjun@gateoverflow.com&gt;
diff --git a/.github/workflows/auto-update-dev.yml b/.github/workflows/auto-update-dev.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master  # Trigger workflow on commits to 'master' branch
+      
 
 jobs:
   update-dev:
diff --git a/language/llama3.1-8b/ref_eval.py b/language/llama3.1-8b/ref_eval.py
@@ -17,7 +17,8 @@ def rouge(label, pred):
 
 
 def niah_em(label, pred):
-    label_uuids = re.findall(r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', label)
+    label_uuids = re.findall(
+        r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', label)
     pred_uuids = re.findall(r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', pred)
 
     # https://github.com/hsiehjackson/RULER/blob/main/scripts/eval/synthetic/constants.py#L28
@@ -43,7 +44,8 @@ def qa_em(label, pred):
         return {'exact_match': 100.0}
 
     normalized_answer = re.sub(r'\s+', '', answer_substring).lower()
-    label_entries = [re.sub(r'\s+', '', entry).lower() for entry in label.split('|')]
+    label_entries = [re.sub(r'\s+', '', entry).lower()
+                     for entry in label.split('|')]
 
     match_found = any(entry in normalized_answer for entry in label_entries)
     return {'exact_match': 100.0 if match_found else 0.0}
@@ -63,7 +65,12 @@ def process_row(row):
 
 def run_evaluation(df):
     with Pool(cpu_count()) as pool:
-        accuracies = list(tqdm(pool.imap(process_row, df.to_dict('records')), total=len(df)))
+        accuracies = list(
+            tqdm(
+                pool.imap(
+                    process_row,
+                    df.to_dict('records')),
+                total=len(df)))
 
     df['accuracy'] = accuracies
     return df
@@ -74,10 +81,10 @@ def run_evaluation(df):
     df = pd.read_pickle(fname)
 
     df = run_evaluation(df)
-    #df.to_pickle(str(fname).replace(".pkl", "_eval.pkl"))
+    # df.to_pickle(str(fname).replace(".pkl", "_eval.pkl"))
     print(f"WROTE: {str(fname).replace('.pkl', '_eval.pkl')}")
 
     accuracy = df.accuracy.apply(pd.Series)
     print(df.dataset.value_counts())
     print(accuracy.describe())
-    print(df.describe())
+    print(df.describe())
diff --git a/loadgen/mlperf.conf b/loadgen/mlperf.conf
@@ -27,9 +27,9 @@ whisper.*.performance_sample_count_override = 1633
 3d-unet.*.performance_sample_count_override = 0
 
 # Set seeds.
-*.*.qsl_rng_seed = 6023615788873153749
-*.*.sample_index_rng_seed = 15036839855038426416
-*.*.schedule_rng_seed = 9933818062894767841
+*.*.qsl_rng_seed = 1780908523862526354
+*.*.sample_index_rng_seed = 14771362308971278857
+*.*.schedule_rng_seed = 18209322760996052031
 
 # Set seeds for TEST_05 (not needed from v5.0 onwards)
 *.*.test05_qsl_rng_seed = 7975553102935885558
diff --git a/text_to_image/tools/sample_ids.py b/text_to_image/tools/sample_ids.py
@@ -20,7 +20,7 @@ def get_args():
         default=10,
         help="Dataset download location")
     parser.add_argument(
-        "--seed", "-s", type=int, default=265673710, help="Dataset download location"
+        "--seed", "-s", type=int, default=927722784, help="Dataset download location"
     )
     args = parser.parse_args()
     return args
diff --git a/text_to_image/tools/sample_ids.txt b/text_to_image/tools/sample_ids.txt
@@ -1,10 +1,10 @@
-716
-2387
-3694
-273
-1221
-2942
-2047
-1691
-1510
-4399
+2747
+2235
+2165
+1515
+1538
+1367
+2419
+4629
+3657
+4532
diff --git a/tools/submission/generate_final_report.py b/tools/submission/generate_final_report.py
@@ -135,26 +135,24 @@ def main():
         [
             "resnet",
             "retinanet",
-            "3d-unet-99",
-            "3d-unet-99.9",
-            "rnnt",
             "bert-99",
             "bert-99.9",
             "dlrm-v2-99",
             "dlrm-v2-99.9",
-            "gptj-99",
-            "gptj-99.9",
-            "stable-diffusion-xl",
+            "3d-unet-99",
+            "3d-unet-99.9",
+            "llama3.1-8b",
             "llama2-70b-99",
             "llama2-70b-99.9",
-            "llama2-70b-interactive-99",
-            "llama2-70b-interactive-99.9",
-            "llama3.1-405b",
+            "stable-diffusion-xl",
             "mixtral-8x7b",
-            "pointpainting",
+            "llama3.1-405b",
             "rgat",
+            "pointpainting",
+            "deepseek-r1",
+            "whisper"
         ],
-        ["SingleStream", "MultiStream", "Server", "Offline"],
+        ["SingleStream", "MultiStream", "Server", "Offline", "Interactive"],
         [
             "Latency (ms)",
             "Samples/s",
@@ -165,14 +163,13 @@ def main():
         ],
     ]
 
-    if args.version == "4.1":
+    if args.version == "5.0":
         filter_scenarios = {
             "datacenter": {
                 "resnet": ["Server", "Offline"],
                 "retinanet": ["Server", "Offline"],
-                "rnnt": ["Server", "Offline"],
-                "bert-99": ["Server", "Offline"],
-                "bert-99.9": ["Server", "Offline"],
+                "bert-99": [],
+                "bert-99.9": [],
                 "dlrm-v2-99": ["Server", "Offline"],
                 "dlrm-v2-99.9": ["Server", "Offline"],
                 "3d-unet-99": ["Offline"],
@@ -182,21 +179,32 @@ def main():
                 "stable-diffusion-xl": ["Server", "Offline"],
                 "llama2-70b-99": ["Server", "Offline"],
                 "llama2-70b-99.9": ["Server", "Offline"],
+                "llama2-70b-interactive-99": ["Server", "Offline"],
+                "llama2-70b-interactive-99.9": ["Server", "Offline"],
                 "mixtral-8x7b": ["Server", "Offline"],
+                "rgat": ["Offline"],
+                "llama3.1-405b": ["Offline", "Server"],
+                "pointpainting": []
             },
             "edge": {
                 "resnet": ["SingleStream", "MultiStream", "Offline"],
                 "retinanet": ["SingleStream", "MultiStream", "Offline"],
-                "rnnt": ["SingleStream", "Offline"],
                 "bert-99": ["SingleStream", "Offline"],
-                "bert-99.9": [],
+                "bert-99.9": ["SingleStream", "Offline"],
                 "dlrm-v2-99": [],
                 "dlrm-v2-99.9": [],
                 "3d-unet-99": ["SingleStream", "Offline"],
                 "3d-unet-99.9": ["SingleStream", "Offline"],
+                "llama2-70b-99": [],
+                "llama2-70b-99.9": [],
+                "llama2-70b-interactive-99": [],
+                "llama2-70b-interactive-99.9": [],
+                "llama3.1-405b": [],
                 "gptj-99": ["SingleStream", "Offline"],
                 "gptj-99.9": ["SingleStream", "Offline"],
+                "rgat": [],
                 "stable-diffusion-xl": ["SingleStream", "Offline"],
+                "pointpainting": ["SingleStream"],
             },
         }
     else:
@@ -210,17 +218,16 @@ def main():
                 "dlrm-v2-99.9": ["Server", "Offline"],
                 "3d-unet-99": ["Offline"],
                 "3d-unet-99.9": ["Offline"],
-                "gptj-99": ["Server", "Offline"],
-                "gptj-99.9": ["Server", "Offline"],
                 "stable-diffusion-xl": ["Server", "Offline"],
-                "llama2-70b-99": ["Server", "Offline"],
-                "llama2-70b-99.9": ["Server", "Offline"],
-                "llama2-70b-interactive-99": ["Server", "Offline"],
-                "llama2-70b-interactive-99.9": ["Server", "Offline"],
+                "llama2-70b-99": ["Server", "Offline", "Interactive"],
+                "llama2-70b-99.9": ["Server", "Offline", "Interactive"],
                 "mixtral-8x7b": ["Server", "Offline"],
                 "rgat": ["Offline"],
                 "llama3.1-405b": ["Offline", "Server"],
-                "pointpainting": []
+                "pointpainting": [],
+                "llama3.1-8b": ["Server", "Offline"],
+                "deepseek-r1": ["Server", "Offline"],
+                "whisper": ["Offline"],
             },
             "edge": {
                 "resnet": ["SingleStream", "MultiStream", "Offline"],
@@ -233,14 +240,12 @@ def main():
                 "3d-unet-99.9": ["SingleStream", "Offline"],
                 "llama2-70b-99": [],
                 "llama2-70b-99.9": [],
-                "llama2-70b-interactive-99": [],
-                "llama2-70b-interactive-99.9": [],
                 "llama3.1-405b": [],
-                "gptj-99": ["SingleStream", "Offline"],
-                "gptj-99.9": ["SingleStream", "Offline"],
                 "rgat": [],
                 "stable-diffusion-xl": ["SingleStream", "Offline"],
                 "pointpainting": ["SingleStream"],
+                "llama3.1-8b": ["SingleStream", "Offline"],
+                "whisper": ["Offline"],
             },
         }
 
diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py

Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ def get_args():`
`20`	`20`	`default=10,`
`21`	`21`	`help="Dataset download location")`
`22`	`22`	`parser.add_argument(`
`23`		`- "--seed", "-s", type=int, default=265673710, help="Dataset download location"`
	`23`	`+ "--seed", "-s", type=int, default=927722784, help="Dataset download location"`
`24`	`24`	`)`
`25`	`25`	`args = parser.parse_args()`
`26`	`26`	`return args`
-Original file line number
+Diff line change
@@ @@ -1,10 +1,10 @@ @@
 -716
 -2387
 -3694
 -273
 -1221
 -2942
 -2047
 -1691
 -1510
 -4399
 +2747
 +2235
 +2165
 +1515
 +1538
 +1367
 +2419
 +4629
 +3657
 +4532