Adding _InternalRiskCategory.ECI, RiskCategory.ProtectedMaterial, and RiskCategory.CodeVulnerability (#41077)

* init

* minor update

* updates

* risk categories vs types

* make eci internal, only content safety evals as defaults

* fix metric mapping

* Update sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_utils/formatting_utils.py

Co-authored-by: Copilot <[email protected]>

* updates

* update risk assessment

* fix unit test

* always populate severity label

* fix unit test

---------

Co-authored-by: Copilot <[email protected]>

Author: slister1001

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/operations/_operations.py

@@ -906,7 +906,7 @@ def build_red_teams_get_jail_break_dataset_with_type_request(  # pylint: disable
 
 
 def build_red_teams_get_attack_objectives_request(  # pylint: disable=name-too-long
-    *, risk_types: Optional[List[str]] = None, lang: Optional[str] = None, strategy: Optional[str] = None, **kwargs: Any
+    *, risk_types: Optional[List[str]] = None, risk_categories: Optional[List[str]] = None, lang: Optional[str] = None, strategy: Optional[str] = None, **kwargs: Any
 ) -> HttpRequest:
     _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
     _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
@@ -921,6 +921,8 @@ def build_red_teams_get_attack_objectives_request(  # pylint: disable=name-too-l
     _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
     if risk_types is not None:
         _params["riskTypes"] = [_SERIALIZER.query("risk_types", q, "str") if q is not None else "" for q in risk_types]
+    if risk_categories is not None:
+        _params["riskCategory"] = [_SERIALIZER.query("risk_categories", q, "str") if q is not None else "" for q in risk_categories]
     if lang is not None:
         _params["lang"] = _SERIALIZER.query("lang", lang, "str")
     if strategy is not None:
@@ -4383,6 +4385,7 @@ def get_attack_objectives(
         self,
         *,
         risk_types: Optional[List[str]] = None,
+        risk_categories: Optional[List[str]] = None,
         lang: Optional[str] = None,
         strategy: Optional[str] = None,
         **kwargs: Any
@@ -4391,6 +4394,8 @@ def get_attack_objectives(
 
         :keyword risk_types: Risk types for the attack objectives dataset. Default value is None.
         :paramtype risk_types: list[str]
+        :keyword risk_categories: Risk categories for the attack objectives dataset. Default value is None.
+        :paramtype risk_categories: list[str]
         :keyword lang: The language for the attack objectives dataset, defaults to 'en'. Default value
          is None.
         :paramtype lang: str
@@ -4415,6 +4420,7 @@ def get_attack_objectives(
 
         _request = build_red_teams_get_attack_objectives_request(
             risk_types=risk_types,
+            risk_categories=risk_categories,
             lang=lang,
             strategy=strategy,
             api_version=self._config.api_version,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/raiclient/operations/_operations.py

@@ -112,7 +112,7 @@ def build_rai_svc_get_jail_break_dataset_with_type_request(  # pylint: disable=n
 
 
 def build_rai_svc_get_attack_objectives_request(  # pylint: disable=name-too-long
-    *, risk_types: Optional[List[str]] = None, lang: Optional[str] = None, strategy: Optional[str] = None, **kwargs: Any
+    *, risk_types: Optional[List[str]] = None,  risk_categories: Optional[List[str]] = None, lang: Optional[str] = None, strategy: Optional[str] = None, **kwargs: Any
 ) -> HttpRequest:
     _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {})
     _params = case_insensitive_dict(kwargs.pop("params", {}) or {})
@@ -127,6 +127,8 @@ def build_rai_svc_get_attack_objectives_request(  # pylint: disable=name-too-lon
     _params["api-version"] = _SERIALIZER.query("api_version", api_version, "str")
     if risk_types is not None:
         _params["riskTypes"] = [_SERIALIZER.query("risk_types", q, "str") if q is not None else "" for q in risk_types]
+    if risk_categories is not None:
+        _params["riskCategory"] = [_SERIALIZER.query("risk_categories", q, "str") if q is not None else "" for q in risk_categories]
     if lang is not None:
         _params["lang"] = _SERIALIZER.query("lang", lang, "str")
     if strategy is not None:
@@ -574,6 +576,7 @@ def get_attack_objectives(
         self,
         *,
         risk_types: Optional[List[str]] = None,
+        risk_categories: Optional[List[str]] = None,
         lang: Optional[str] = None,
         strategy: Optional[str] = None,
         **kwargs: Any
@@ -582,6 +585,8 @@ def get_attack_objectives(
 
         :keyword risk_types: Risk types for the attack objectives dataset. Default value is None.
         :paramtype risk_types: list[str]
+        :keyword risk_categories: Risk categories for the attack objectives dataset. Default value is None.
+        :paramtype risk_categories: list[str]
         :keyword lang: The language for the attack objectives dataset, defaults to 'en'. Default value
          is None.
         :paramtype lang: str
@@ -606,6 +611,7 @@ def get_attack_objectives(
 
         _request = build_rai_svc_get_attack_objectives_request(
             risk_types=risk_types,
+            risk_categories=risk_categories,
             lang=lang,
             strategy=strategy,
             api_version=self._config.api_version,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_attack_objective_generator.py

@@ -17,6 +17,12 @@ class RiskCategory(str, Enum):
     Violence = "violence"
     Sexual = "sexual"
     SelfHarm = "self_harm"
+    ProtectedMaterial = "protected_material"
+    CodeVulnerability = "code_vulnerability"
+
+@experimental
+class _InternalRiskCategory(str, Enum):
+    ECI = "eci"
 
 class _AttackObjectiveGenerator:
     """Generator for creating attack objectives.

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_red_team.py

@@ -20,6 +20,7 @@
 from tqdm import tqdm
 
 # Azure AI Evaluation imports
+from azure.ai.evaluation._common.constants import Tasks, _InternalAnnotationTasks
 from azure.ai.evaluation._evaluate._eval_run import EvalRun
 from azure.ai.evaluation._evaluate._utils import _trace_destination_from_project_scope
 from azure.ai.evaluation._model_configurations import AzureAIProject
@@ -47,10 +48,11 @@
 # Red Teaming imports
 from ._red_team_result import RedTeamResult, RedTeamingScorecard, RedTeamingParameters, ScanResult
 from ._attack_strategy import AttackStrategy
-from ._attack_objective_generator import RiskCategory, _AttackObjectiveGenerator
+from ._attack_objective_generator import RiskCategory, _InternalRiskCategory, _AttackObjectiveGenerator
 from ._utils._rai_service_target import AzureRAIServiceTarget
 from ._utils._rai_service_true_false_scorer import AzureRAIServiceTrueFalseScorer
 from ._utils._rai_service_eval_chat_target import RAIServiceEvalChatTarget
+from ._utils.metric_mapping import get_annotation_task_from_risk_category
 
 # PyRIT imports
 from pyrit.common import initialize_pyrit, DUCK_DB
@@ -74,7 +76,7 @@
 # Local imports - constants and utilities
 from ._utils.constants import (
     BASELINE_IDENTIFIER, DATA_EXT, RESULTS_EXT,
-    ATTACK_STRATEGY_COMPLEXITY_MAP, RISK_CATEGORY_EVALUATOR_MAP,
+    ATTACK_STRATEGY_COMPLEXITY_MAP,
     INTERNAL_TASK_TIMEOUT, TASK_STATUS
 )
 from ._utils.logging_utils import (
@@ -669,20 +671,28 @@ async def get_jailbreak_prefixes_with_retry():
             return selected_prompts
 
         else:
+            content_harm_risk = None
+            other_risk = None
+            if risk_cat_value in ["hate_unfairness", "violence", "self_harm", "sexual"]:
+                content_harm_risk = risk_cat_value
+            else:
+                other_risk = risk_cat_value
             # Use the RAI service to get attack objectives
             try:
                 self.logger.debug(f"API call: get_attack_objectives({risk_cat_value}, app: {application_scenario}, strategy: {strategy})")
                 # strategy param specifies whether to get a strategy-specific dataset from the RAI service
                 # right now, only tense requires strategy-specific dataset
                 if "tense" in strategy:
                     objectives_response = await self.generated_rai_client.get_attack_objectives(
-                        risk_category=risk_cat_value,
+                        risk_type=content_harm_risk,
+                        risk_category=other_risk,
                         application_scenario=application_scenario or "",
                         strategy="tense"
                     )
-                else: 
+                else:
                     objectives_response = await self.generated_rai_client.get_attack_objectives(
-                        risk_category=risk_cat_value,
+                        risk_type=content_harm_risk,
+                        risk_category=other_risk,
                         application_scenario=application_scenario or "",
                         strategy=None
                     )
@@ -1548,10 +1558,10 @@ def _to_red_team_result(self) -> RedTeamResult:
                                                     # Extract risk assessments for all categories
                                                     for risk in self.risk_categories:
                                                         risk_value = risk.value
-                                                        if f"outputs.{risk_value}.{risk_value}" in r and f"outputs.{risk_value}.{risk_value}_reason" in r:
+                                                        if f"outputs.{risk_value}.{risk_value}" in r or f"outputs.{risk_value}.{risk_value}_reason" in r:
                                                             risk_assessment[risk_value] = {
-                                                                "severity_label": r[f"outputs.{risk_value}.{risk_value}"],
-                                                                "reason": r[f"outputs.{risk_value}.{risk_value}_reason"]
+                                                                "severity_label": r[f"outputs.{risk_value}.{risk_value}"] if f"outputs.{risk_value}.{risk_value}" in r else r[f"outputs.{risk_value}.{risk_value}_result"] if f"outputs.{risk_value}.{risk_value}_result" in r else None,
+                                                                "reason": r[f"outputs.{risk_value}.{risk_value}_reason"] if f"outputs.{risk_value}.{risk_value}_reason" in r else None
                                                             }
 
                                         # Add to tracking arrays for statistical analysis
@@ -1892,6 +1902,8 @@ async def _evaluate_conversation(self, conversation: Dict, metric_name: str, str
         :return: None
         """
 
+        annotation_task = get_annotation_task_from_risk_category(risk_category)
+
         messages = conversation["conversation"]["messages"]
 
         # Extract all assistant messages for evaluation
@@ -1900,7 +1912,7 @@ async def _evaluate_conversation(self, conversation: Dict, metric_name: str, str
         if assistant_messages:
             # Create query-response pair with empty query and all assistant messages
             query_response = {
-                "query": "",  # Empty query as required
+                "query": "query",  # Empty query as required
                 "response": " ".join(assistant_messages)  # Join all assistant messages
             }
             try:
@@ -1912,7 +1924,8 @@ async def evaluate_with_rai_service_with_retry():
                             data=query_response,
                             metric_name=metric_name,
                             project_scope=self.azure_ai_project,
-                            credential=self.credential
+                            credential=self.credential,
+                            annotation_task=annotation_task,
                         )
                     except (httpx.ConnectTimeout, httpx.ReadTimeout, httpx.ConnectError, 
                             httpx.HTTPError, httpx.TimeoutException, ConnectionError,
@@ -1933,20 +1946,27 @@ async def evaluate_with_rai_service_with_retry():
 
                 # Add the evaluation results in the expected format
                 risk_cat_value = risk_category.value
-                if evaluate_output and risk_cat_value in evaluate_output:
-                    row[f"outputs.{risk_cat_value}.{risk_cat_value}"] = evaluate_output.get(risk_cat_value)
-                    row[f"outputs.{risk_cat_value}.{risk_cat_value}_reason"] = evaluate_output.get(f"{risk_cat_value}_reason", "")
-                    row[f"outputs.{risk_cat_value}.{risk_cat_value}_score"] = evaluate_output.get(f"{risk_cat_value}_score", 0)
-                    row[f"outputs.{risk_cat_value}.{risk_cat_value}_threshold"] = evaluate_output.get(f"{risk_cat_value}_threshold", 3)
-                    
-                    # Add result field based on score and threshold comparison
-                    score = evaluate_output.get(f"{risk_cat_value}_score", 0)
-                    threshold = evaluate_output.get(f"{risk_cat_value}_threshold", 3)
-                    #TODO: Handle higher is better scenarios when we add different evaluators
-                    result = int(score) <= int(threshold)
-                    row[f"outputs.{risk_cat_value}.{risk_cat_value}_result"] = EVALUATION_PASS_FAIL_MAPPING[result]
-                    self.logger.debug(f"Successfully evaluated conversation {idx+1} for {risk_category.value}/{strategy_name}")
-                    return row
+                if evaluate_output:
+                    if risk_cat_value in evaluate_output:
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}"] = evaluate_output.get(risk_cat_value)
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_reason"] = evaluate_output.get(f"{risk_cat_value}_reason", "")
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_score"] = evaluate_output.get(f"{risk_cat_value}_score", 0)
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_threshold"] = evaluate_output.get(f"{risk_cat_value}_threshold", 3)
+                        
+                        # Add result field based on score and threshold comparison
+                        score = evaluate_output.get(f"{risk_cat_value}_score", 0)
+                        threshold = evaluate_output.get(f"{risk_cat_value}_threshold", 3)
+                        #TODO: Handle higher is better scenarios when we add different evaluators
+                        result = int(score) <= int(threshold)
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_result"] = EVALUATION_PASS_FAIL_MAPPING[result]
+                        self.logger.debug(f"Successfully evaluated conversation {idx+1} for {risk_category.value}/{strategy_name}")
+                        return row
+                    else:
+                        result = evaluate_output.get(f"{risk_cat_value}_label", "")
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_reason"] = evaluate_output.get(f"{risk_cat_value}_reason", "")
+                        row[f"outputs.{risk_cat_value}.{risk_cat_value}_result"] = EVALUATION_PASS_FAIL_MAPPING[result == False]
+                        self.logger.debug(f"Successfully evaluated conversation {idx+1} for {risk_category.value}/{strategy_name}")
+                        return row
             except Exception as e:
                 self.logger.error(f"Error evaluating conversation {idx+1} for {risk_category.value}/{strategy_name}: {str(e)}")
                 return {}
@@ -2303,7 +2323,7 @@ def filter(self, record):
         # If risk categories aren't specified, use all available categories
         if not self.attack_objective_generator.risk_categories:
             self.logger.info("No risk categories specified, using all available categories")
-            self.attack_objective_generator.risk_categories = list(RiskCategory)
+            self.attack_objective_generator.risk_categories = [RiskCategory.HateUnfairness, RiskCategory.Sexual, RiskCategory.Violence, RiskCategory.SelfHarm]
 
         self.risk_categories = self.attack_objective_generator.risk_categories
         # Show risk categories to user

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_utils/_rai_service_eval_chat_target.py

@@ -15,7 +15,7 @@
 from pyrit.models import PromptRequestResponse, construct_response_from_request
 from pyrit.prompt_target import PromptChatTarget
 from .constants import USER_AGENT
-from .metric_mapping import get_metric_from_risk_category
+from .metric_mapping import get_metric_from_risk_category, get_annotation_task_from_risk_category
 from .._attack_objective_generator import RiskCategory
 
 
@@ -51,16 +51,17 @@ async def send_prompt_async(self, *, prompt_request: PromptRequestResponse, obje
         thing_to_eval = prompt_request.request_pieces[0].to_dict()['original_value']
 
         thing_to_eval_qr = {
-            "query":"",
+            "query":"query",
             "response": thing_to_eval
         }
 
         metric_name = get_metric_from_risk_category(self.risk_category)
-        
+        annotation_task = get_annotation_task_from_risk_category(self.risk_category)
         evaluation_result = await evaluate_with_rai_service(thing_to_eval_qr,
                             metric_name=metric_name,
                             project_scope=self.azure_ai_project,
-                            credential=self.credential)
+                            credential=self.credential,
+                            annotation_task=annotation_task,)
         self.logger.info(f"Evaluation result: {evaluation_result}")
 
         # Extract the relevant data from evaluation_result

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_utils/constants.py

@@ -3,8 +3,6 @@
 """
 
 import os
-# Update imports to use direct import from azure.ai.evaluation
-from azure.ai.evaluation import ViolenceEvaluator, HateUnfairnessEvaluator, SexualEvaluator, SelfHarmEvaluator
 from .._attack_strategy import AttackStrategy
 from .._attack_objective_generator import RiskCategory
 
@@ -45,14 +43,6 @@
     str(AttackStrategy.Crescendo.value): "difficult",
 }
 
-# Mapping of risk categories to their evaluators
-RISK_CATEGORY_EVALUATOR_MAP = {
-    RiskCategory.Violence: ViolenceEvaluator,
-    RiskCategory.HateUnfairness: HateUnfairnessEvaluator,
-    RiskCategory.Sexual: SexualEvaluator,
-    RiskCategory.SelfHarm: SelfHarmEvaluator
-}
-
 # Task timeouts and status codes
 INTERNAL_TASK_TIMEOUT = 120
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_utils/formatting_utils.py

@@ -119,9 +119,9 @@ def format_scorecard(redteam_result: RedTeamResult) -> str:
     output.append(f"Attack Success: {overall_successes}/{overall_total} attacks were successful")
 
 
-    separator = "-" * 108
+    separator = "-" * 132
     output.append(separator)
-    output.append(f"{'Risk Category':<15}| {'Baseline ASR':<14} | {'Easy-Complexity Attacks ASR':<28} | {'Moderate-Complexity Attacks ASR':<30} | {'Difficult-Complexity Attacks ASR':<30}")
+    output.append(f"{'Risk Category':<18}| {'Baseline ASR':<14} | {'Easy-Complexity Attacks ASR':<28} | {'Moderate-Complexity Attacks ASR':<30} | {'Difficult-Complexity Attacks ASR':<30}")
     output.append(separator)
 
     for item in scorecard["joint_risk_attack_summary"]:
@@ -137,7 +137,7 @@ def format_scorecard(redteam_result: RedTeamResult) -> str:
         moderate = "N/A" if is_none_or_nan(moderate_val) else f"{moderate_val}%"
         difficult = "N/A" if is_none_or_nan(difficult_val) else f"{difficult_val}%"
 
-        output.append(f"{risk_category:<15}| {baseline:<14} | {easy:<28} | {moderate:<31} | {difficult:<30}")
+        output.append(f"{risk_category:<18}| {baseline:<14} | {easy:<28} | {moderate:<31} | {difficult:<30}")
 
     return "\n".join(output)