feat: generate samples working

Author: Jon Saad-Falcon

.vscode/launch.json

@@ -25,7 +25,7 @@
             "request": "launch",
             "program": "scripts/generate_samples.py",
             "args": [
-                "run_name=test_hf_level_1",
+                "run_name=test_hf_archon_level_1",
                 "dataset_src=huggingface",
                 "level=1",
                 "num_workers=50",
@@ -42,7 +42,7 @@
             "program": "scripts/eval_from_generations.py",
             "args": [
                 "level=1",
-                "run_name=test_hf_level_1",
+                "run_name=test_hf_archon_level_1",
                 "dataset_src=local",
                 "num_gpu_devices=8",
                 "timeout=300"

scripts/generate_samples.py

@@ -190,6 +190,7 @@ def main(config: GenerationConfig):
                                                         model_name=config.model_name,
                                                         temperature=config.temperature,
                                                         max_tokens=config.max_tokens,
+                                                        archon_config_path=config.archon_config_path,
                                                         verbose=config.verbose)
 
     # Launch workers

src/utils.py

@@ -147,6 +147,8 @@ def query_server(
             client = OpenAI(api_key=OPENAI_KEY)
             model = model_name
         case "archon":
+            assert archon_config_path is not None, "Archon config path is required"
+            assert os.path.exists(archon_config_path), f"Archon config path {archon_config_path} does not exist"
             client = Archon(json.load(open(archon_config_path)))
             model = model_name
         case _: