added AnswerCorrectness and AnswerFaithfulness auto-eval modules

Author: arnavsinghvi11

dspy/evaluate/__init__.py

@@ -1,2 +1,3 @@
 from .evaluate import Evaluate
 from .metrics import *
+from .auto_evaluation import *

dspy/evaluate/auto_evaluation.py

@@ -0,0 +1,34 @@
+import dspy 
+
+class AnswerCorrectnessSignature(dspy.Signature):
+    """Determines if predicted answer matches the gold answer."""
+
+    question = dspy.InputField()
+    gold_answer = dspy.InputField(desc="correct answer for question")
+    predicted_answer = dspy.InputField(desc="predicted answer for question")
+    is_correct = dspy.OutputField(desc='True or False')
+
+class AnswerCorrectness(dspy.Module):
+    def __init__(self):
+        super().__init__()
+        self.evaluate_correctness = dspy.ChainOfThought(AnswerCorrectnessSignature)
+    
+    def forward(self, question, gold_answer, predicted_answer):
+        return self.evaluate_correctness(question=question, gold_answer=gold_answer, predicted_answer=predicted_answer)
+
+
+class AnswerFaithfulnessSignature(dspy.Signature):
+    """Checks if answer for question is based on rationale."""
+    
+    context = dspy.InputField(desc="relevant facts for producing answer")
+    question = dspy.InputField()
+    answer = dspy.InputField(desc="often between 1 and 5 words")
+    faithful = dspy.OutputField(desc='True or False')
+
+class AnswerFaithfulness(dspy.Module):
+    def __init__(self):
+        super().__init__()
+        self.evaluate_faithfulness = dspy.ChainOfThought(AnswerFaithfulnessSignature)
+    
+    def forward(self, context, question, answer):
+        return self.evaluate_faithfulness(context=context, question=question, answer=answer)

dspy/signatures/signature.py

@@ -33,6 +33,10 @@ def __new__(cls, name, bases, class_dict):
 
         return new_class
 
+    @property
+    def kwargs(cls):
+        return cls.signature.fields
+    
     def __call__(cls, *args, **kwargs):
         if len(args) == 1 and isinstance(args[0], str):
             instance = super(SignatureMeta, cls).__call__(*args, **kwargs)
@@ -42,7 +46,9 @@ def __call__(cls, *args, **kwargs):
 
     def __getattr__(cls, attr):
         # Redirect attribute access to the template object when accessed on the class directly
-        return getattr(cls._template, attr)
+        if attr not in cls.__dict__:
+            return getattr(cls._template, attr)
+        return super().__getattr__(attr)
 
 class Signature(metaclass=SignatureMeta):
     def __init__(self, signature: str = "", instructions: str = ""):
@@ -51,6 +57,11 @@ def __init__(self, signature: str = "", instructions: str = ""):
         self.fields = {}
         self.parse_structure()
 
+    def __getattr__(self, attr):
+        if attr not in self.__dict__:
+            return getattr(self.__class__, attr)
+        return super().__getattr__(attr)
+
     @property
     def kwargs(self):
         return {k: v for k, v in self.fields.items()}