Skip to content

Commit 577ddc8

Browse files
add test to ensure saving backward compatibility (stanfordnlp#8680)
1 parent c698547 commit 577ddc8

File tree

2 files changed

+125
-6
lines changed

2 files changed

+125
-6
lines changed
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"react": {
3+
"traces": [],
4+
"train": [],
5+
"demos": [
6+
{
7+
"augmented": true,
8+
"question": "That Darn Cat! and Never a Dull Moment were both produced by what studio?",
9+
"trajectory": "[[ ## thought_0 ## ]]\nI need to find out which studio produced both \"That Darn Cat!\" and \"Never a Dull Moment.\" This information is likely available on Wikipedia, so I will search for it there.\n\n[[ ## tool_name_0 ## ]]\nsearch_wikipedia\n\n[[ ## tool_args_0 ## ]]\n{\"query\": \"That Darn Cat! and Never a Dull Moment studio production\"}\n\n[[ ## observation_0 ## ]]\n[1] «That Darn Cat! | That Darn Cat! is a 1965 American Walt Disney Productions thriller comedy film starring Hayley Mills (in her last of the six films she made for the Walt Disney Studios) and Dean Jones (starring in his first film for Disney) in a story about bank robbers, a kidnapping and a mischievous cat. The film was based on the 1963 novel \"Undercover Cat\" by Gordon and Mildred Gordon and was directed by Robert Stevenson. The title song was written by the Sherman Brothers and sung by Bobby Darin. The 1997 remake includes a cameo appearance by Dean Jones.»\n[2] «Never a Dull Moment (1968 film) | Never a Dull Moment is a 1968 American comedy crime film from Walt Disney Productions starring Dick Van Dyke and Edward G. Robinson and directed by Jerry Paris. The script by AJ Carothers was based on a novel by John Godey. The supporting cast features Dorothy Provine, Henry Silva, Slim Pickens and Jack Elam.»\n[3] «That Darn Cat (1997 film) | That Darn Cat is a 1997 American mystery comedy film starring Christina Ricci and Doug E. Doug. It is a remake of the 1965 film \"That Darn Cat! \", which in turn was based on the book \"Undercover Cat\" by Gordon and Mildred Gordon. It is directed by British TV veteran Bob Spiers (most famous for \"Fawlty Towers\", as well as \"Spice World\") and written by Scott Alexander and Larry Karaszewski, best known for \"Ed Wood\" and the first two \"Problem Child\" films.»",
10+
"next_thought": "Both \"That Darn Cat!\" and \"Never a Dull Moment\" were produced by Walt Disney Productions. I have the information I need to answer the question.",
11+
"next_tool_name": "finish",
12+
"next_tool_args": {}
13+
},
14+
{
15+
"question": "Was Yakov Protazanov or Marcel Duchamp born in 1881",
16+
"answer": "Yakov Alexandrovich Protazanov (Russian: Я́ков Алекса́ндрович Протаза́нов ; January 23 (O.S. February 4), 1881",
17+
"dspy_uuid": "15118a69-274d-4eea-b3eb-204d05e24012",
18+
"dspy_split": "train"
19+
}
20+
],
21+
"signature": {
22+
"instructions": "Imagine you are a detective racing against time to solve a high-profile case involving two famous films: \"That Darn Cat!\" and \"Never a Dull Moment.\" Your mission is to uncover which studio produced these films before the press conference starts in one hour. You have access to a powerful tool: a Wikipedia search. \n\nGiven the fields `question`, produce the fields `answer`.\n\nYou are an Agent. In each episode, you will be given the fields `question` as input. And you can see your past trajectory so far. Your goal is to use one or more of the supplied tools to collect any necessary information for producing `answer`.\n\nTo do this, you will interleave next_thought, next_tool_name, and next_tool_args in each turn, and also when finishing the task. After each tool call, you receive a resulting observation, which gets appended to your trajectory.\n\nWhen writing next_thought, you may reason about the current situation and plan for future steps. When selecting the next_tool_name and its next_tool_args, the tool must be one of:\n\n(1) search_wikipedia. It takes arguments {'query': {'type': 'string'}}.\n(2) finish, whose description is <desc>Marks the task as complete. That is, signals that all information for producing the outputs, i.e. `answer`, are now available to be extracted.<\/desc>. It takes arguments {}.\nWhen providing `next_tool_args`, the value inside the field must be in JSON format.",
23+
"fields": [
24+
{
25+
"prefix": "Question:",
26+
"description": "${question}"
27+
},
28+
{
29+
"prefix": "Trajectory:",
30+
"description": "${trajectory}"
31+
},
32+
{
33+
"prefix": "Next Thought:",
34+
"description": "${next_thought}"
35+
},
36+
{
37+
"prefix": "Next Tool Name:",
38+
"description": "${next_tool_name}"
39+
},
40+
{
41+
"prefix": "Next Tool Args:",
42+
"description": "${next_tool_args}"
43+
}
44+
]
45+
},
46+
"lm": null
47+
},
48+
"extract.predict": {
49+
"traces": [],
50+
"train": [],
51+
"demos": [
52+
{
53+
"augmented": true,
54+
"question": "That Darn Cat! and Never a Dull Moment were both produced by what studio?",
55+
"trajectory": "[[ ## thought_0 ## ]]\nI need to find out which studio produced both \"That Darn Cat!\" and \"Never a Dull Moment.\" This information is likely available on Wikipedia, so I will search for it there.\n\n[[ ## tool_name_0 ## ]]\nsearch_wikipedia\n\n[[ ## tool_args_0 ## ]]\n{\"query\": \"That Darn Cat! and Never a Dull Moment studio production\"}\n\n[[ ## observation_0 ## ]]\n[1] «That Darn Cat! | That Darn Cat! is a 1965 American Walt Disney Productions thriller comedy film starring Hayley Mills (in her last of the six films she made for the Walt Disney Studios) and Dean Jones (starring in his first film for Disney) in a story about bank robbers, a kidnapping and a mischievous cat. The film was based on the 1963 novel \"Undercover Cat\" by Gordon and Mildred Gordon and was directed by Robert Stevenson. The title song was written by the Sherman Brothers and sung by Bobby Darin. The 1997 remake includes a cameo appearance by Dean Jones.»\n[2] «Never a Dull Moment (1968 film) | Never a Dull Moment is a 1968 American comedy crime film from Walt Disney Productions starring Dick Van Dyke and Edward G. Robinson and directed by Jerry Paris. The script by AJ Carothers was based on a novel by John Godey. The supporting cast features Dorothy Provine, Henry Silva, Slim Pickens and Jack Elam.»\n[3] «That Darn Cat (1997 film) | That Darn Cat is a 1997 American mystery comedy film starring Christina Ricci and Doug E. Doug. It is a remake of the 1965 film \"That Darn Cat! \", which in turn was based on the book \"Undercover Cat\" by Gordon and Mildred Gordon. It is directed by British TV veteran Bob Spiers (most famous for \"Fawlty Towers\", as well as \"Spice World\") and written by Scott Alexander and Larry Karaszewski, best known for \"Ed Wood\" and the first two \"Problem Child\" films.»\n\n[[ ## thought_1 ## ]]\nBoth \"That Darn Cat!\" and \"Never a Dull Moment\" were produced by Walt Disney Productions. I have the information I need to answer the question.\n\n[[ ## tool_name_1 ## ]]\nfinish\n\n[[ ## tool_args_1 ## ]]\n{}\n\n[[ ## observation_1 ## ]]\nCompleted.",
56+
"reasoning": "Both \"That Darn Cat!\" and \"Never a Dull Moment\" were produced by Walt Disney Productions, as confirmed by the information retrieved from Wikipedia.",
57+
"answer": "Walt Disney Productions"
58+
},
59+
{
60+
"question": "Are Smyrnium and Nymania both types of plant?",
61+
"answer": "yes",
62+
"dspy_uuid": "b57b5933-95c7-472a-801b-3cc9bc0a3b99",
63+
"dspy_split": "train"
64+
}
65+
],
66+
"signature": {
67+
"instructions": "Given the very verbose fields `question`, produce the fields `answer`.",
68+
"fields": [
69+
{
70+
"prefix": "Question:",
71+
"description": "${question}"
72+
},
73+
{
74+
"prefix": "Trajectory:",
75+
"description": "${trajectory}"
76+
},
77+
{
78+
"prefix": "Reasoning: Let's think step by step in order to",
79+
"description": "${reasoning}"
80+
},
81+
{
82+
"prefix": "Answer:",
83+
"description": "${answer}"
84+
}
85+
]
86+
},
87+
"lm": null
88+
},
89+
"metadata": {
90+
"dependency_versions": {
91+
"python": "3.13",
92+
"dspy": "3.0.0",
93+
"cloudpickle": "3.1"
94+
}
95+
}
96+
}

tests/primitives/test_module.py

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from pathlib import Path
2+
13
import dspy
24
from dspy.primitives.module import Module, set_attribute_by_name # Adjust the import based on your file structure
35
from dspy.utils import DummyLM
@@ -114,9 +116,9 @@ def test_complex_module_traversal():
114116
}
115117
found_names = {name for name, _ in root.named_sub_modules()}
116118

117-
assert (
118-
found_names == expected_names
119-
), f"Missing or extra modules found. Missing: {expected_names-found_names}, Extra: {found_names-expected_names}"
119+
assert found_names == expected_names, (
120+
f"Missing or extra modules found. Missing: {expected_names - found_names}, Extra: {found_names - expected_names}"
121+
)
120122

121123

122124
def test_complex_module_traversal_with_same_module():
@@ -135,9 +137,9 @@ def test_complex_module_traversal_with_same_module():
135137
}
136138
found_names = {name for name, _ in root.named_sub_modules()}
137139

138-
assert (
139-
found_names == expected_names
140-
), f"Missing or extra modules found. Missing: {expected_names-found_names}, Extra: {found_names-expected_names}"
140+
assert found_names == expected_names, (
141+
f"Missing or extra modules found. Missing: {expected_names - found_names}, Extra: {found_names - expected_names}"
142+
)
141143

142144

143145
def test_complex_module_set_attribute_by_name():
@@ -174,3 +176,24 @@ def test_named_parameters_duplicate_references():
174176
# Only testing for whether exceptions are thrown or not
175177
# As Module.named_parameters() is recursive, this is mainly for catching infinite recursion
176178
module.named_parameters()
179+
180+
181+
def test_load_dspy_program_cross_version():
182+
"""
183+
Test backward compatibility for loading a saved DSPy program.
184+
185+
This test verifies that DSPy can load a program saved in version 3.0.1, ensuring compatibility with older versions.
186+
The saved state is located in 'test/primitives/resources/saved_program.json' and represents an optimized
187+
`dspy.ReAct` program.
188+
"""
189+
path = Path(__file__).parent / "resources" / "saved_program.json"
190+
loaded_react = dspy.ReAct("question->answer", tools=[])
191+
loaded_react.load(path)
192+
assert (
193+
"Imagine you are a detective racing against time to solve a high-profile"
194+
in loaded_react.react.signature.instructions
195+
)
196+
assert "Given the very verbose fields `question`" in loaded_react.extract.predict.signature.instructions
197+
198+
assert len(loaded_react.react.demos) == 2
199+
assert len(loaded_react.extract.predict.demos) == 2

0 commit comments

Comments
 (0)