removed factcheck input guardrail

Author: abhijitpal1247

examples/configs/autoguard/autoguard_factcheck_config/config.yml

@@ -7,9 +7,6 @@ rails:
     autoguard:
       parameters:
         fact_check_endpoint: "https://nvidia.autoalign.ai/factcheck"
-  input:
-    flows:
-      - autoguard factcheck input
   output:
     flows:
       - autoguard factcheck output

nemoguardrails/library/autoguard/README.md

@@ -563,18 +563,15 @@ rails:
     autoguard:
       parameters:
         fact_check_endpoint: "https://nvidia.autoalign.ai/factcheck"
-  input:
-    flows:
-      - autoguard factcheck input
   output:
     flows:
       - autoguard factcheck output
 ```
 
 Specify the factcheck endpoint the parameters section of autoguard's config.
-Then, you have to call the corresponding subflows for input and output factcheck guardrails.
+Then, you have to call the corresponding subflows for factcheck guardrails.
 
-Following is the format of the colang file:
+Following is the format of the colang file, which is present in the library:
 ```colang
 define flow autoguard factcheck input
   execute autoguard_retrieve_relevant_chunks_input
@@ -595,9 +592,6 @@ define bot inform autoguard factcheck output violation
   "$bot_message Factcheck output violation has been detected by AutoGuard."
 ```
 
-Within the flow you can see we have an action for custom relevant chunk extraction, `autoguard_retrieve_relevant_chunks_input`,
-which ensures that the documents are passed in the context for the guardrail while using it for user input.
-
 The output of the factcheck endpoint provides you with a factcheck score against which we can add a threshold which determines whether the given output is factually correct or not.
 
 The supporting documents or the evidence has to be placed within a `kb` folder within `config` folder.

nemoguardrails/library/autoguard/actions.py

@@ -265,32 +265,6 @@ async def autoguard_output_api(
     return autoguard_response
 
 
-@action(name="autoguard_factcheck_input_api")
-async def autoguard_factcheck_input_api(
-    llm_task_manager: LLMTaskManager, context: Optional[dict] = None
-):
-    """Calls AutoGuard factcheck API and checks whether the user message is factually correct according to given
-    documents"""
-
-    user_message = context.get("user_message")
-    documents = context.get("relevant_chunks", [])
-    autoguard_config = llm_task_manager.config.rails.config.autoguard
-    autoguard_fact_check_api_url = autoguard_config.parameters.get(
-        "fact_check_endpoint"
-    )
-    if not autoguard_fact_check_api_url:
-        raise ValueError("Provide the autoguard factcheck endpoint in the config")
-    if isinstance(documents, str):
-        documents = documents.split("\n")
-    prompt = user_message
-    if isinstance(documents, list) and len(documents) > 0:
-        return await autoguard_factcheck_infer(
-            autoguard_fact_check_api_url, prompt, documents
-        )
-    else:
-        raise ValueError("Provide relevant documents in proper format")
-
-
 @action(name="autoguard_factcheck_output_api")
 async def autoguard_factcheck_output_api(
     llm_task_manager: LLMTaskManager, context: Optional[dict] = None
@@ -315,24 +289,3 @@ async def autoguard_factcheck_output_api(
         )
     else:
         raise ValueError("Provide relevant documents in proper format")
-
-
-@action(name="autoguard_retrieve_relevant_chunks_input")
-async def autoguard_retrieve_relevant_chunks_input(
-    context: Optional[dict] = None,
-    kb: Optional[KnowledgeBase] = None,
-):
-    """Retrieve knowledge chunks from knowledge base and update the context."""
-    user_message = context.get("user_message")
-    context_updates = {}
-    chunks = await kb.search_relevant_chunks(user_message)
-    chunks = [chunk["body"] for chunk in chunks]
-    # 💡 Store the chunks for fact-checking
-
-    context_updates["relevant_chunks"] = "\n".join(chunks)
-    context_updates["relevant_chunks_sep"] = chunks
-
-    return ActionResult(
-        return_value=context_updates["relevant_chunks"],
-        context_updates=context_updates,
-    )

nemoguardrails/library/autoguard/flows.co

@@ -1,14 +1,12 @@
 define flow autoguard check input
   $input_result = execute autoguard_input_api(show_autoguard_message=True)
-
   if $input_result["guardrails_triggered"]
     $autoguard_input_response = $input_result['combined_response']
     bot refuse to respond
     stop
 
 define flow autoguard check output
   $output_result = execute autoguard_output_api(show_autoguard_message=True)
-
   if $output_result["guardrails_triggered"]
     bot refuse to respond
     stop
@@ -18,21 +16,12 @@ define flow autoguard check output
       bot respond pii output
       stop
 
-define flow autoguard factcheck input
-  execute autoguard_retrieve_relevant_chunks_input
-  $input_result = execute autoguard_factcheck_input_api
-
 define flow autoguard factcheck output
   $output_result = execute autoguard_factcheck_output_api
-  if $input_result < 0.5
-    bot inform autoguard factcheck input violation
   if $output_result < 0.5
     bot inform autoguard factcheck output violation
     stop
 
-define bot inform autoguard factcheck input violation
-  "Factcheck violation in user input has been detected by AutoGuard."
-
 define bot inform autoguard factcheck output violation
   "Factcheck violation in llm response has been detected by AutoGuard."
 

tests/test_autoguard_factcheck.py

@@ -47,19 +47,6 @@ async def retrieve_relevant_chunks():
     )
 
 
-@action(is_system_action=True)
-async def autoguard_retrieve_relevant_chunks_input():
-    """Retrieve relevant chunks from the knowledge base and add them to the context."""
-    context_updates = {}
-    relevant_chunks = "\n".join(build_kb())
-    context_updates["relevant_chunks"] = relevant_chunks
-
-    return ActionResult(
-        return_value=context_updates["relevant_chunks"],
-        context_updates=context_updates,
-    )
-
-
 @pytest.mark.asyncio
 async def test_fact_checking_correct(httpx_mock):
     config = RailsConfig.from_path(os.path.join(CONFIGS_FOLDER, "autoguard_factcheck"))
@@ -75,19 +62,6 @@ async def test_fact_checking_correct(httpx_mock):
         ],
     )
 
-    async def mock_autoguard_factcheck_input_api(
-        context: Optional[dict] = None, **kwargs
-    ):
-        query = context.get("user_message")
-        if (
-            query
-            == "Pluto, with its eccentric orbit, comes closer to the Sun than Neptune at times, yet a stable "
-            "orbital resonance ensures they do not collide."
-        ):
-            return 1.0
-        else:
-            return 0.0
-
     async def mock_autoguard_factcheck_output_api(
         context: Optional[dict] = None, **kwargs
     ):
@@ -106,9 +80,6 @@ async def mock_autoguard_factcheck_output_api(
         else:
             return 0.0
 
-    chat.app.register_action(
-        mock_autoguard_factcheck_input_api, "autoguard_factcheck_input_api"
-    )
     chat.app.register_action(
         mock_autoguard_factcheck_output_api, "autoguard_factcheck_output_api"
     )
@@ -144,23 +115,6 @@ async def test_fact_checking_wrong(httpx_mock):
         ],
     )
 
-    async def mock_autoguard_factcheck_input_api(
-        context: Optional[dict] = None, **kwargs
-    ):
-        query = context.get("user_message")
-        if (
-            query
-            == "Pluto has no known moons; Charon, the smallest, has a diameter greater than Pluto's, along with "
-            "the non-existent Styx, Nix, Kerberos, and Hydra."
-        ):
-            return 0.0
-        else:
-            return 1.0
-
-    chat.app.register_action(
-        mock_autoguard_factcheck_input_api, "autoguard_factcheck_input_api"
-    )
-
     async def mock_autoguard_factcheck_output_api(
         context: Optional[dict] = None, **kwargs
     ):
@@ -186,6 +140,5 @@ async def mock_autoguard_factcheck_output_api(
         "non-existent Styx, Nix, Kerberos, and Hydra."
     )
     await chat.bot_async(
-        "Factcheck violation in user input has been detected by AutoGuard.\nFactcheck violation in "
-        "llm response has been detected by AutoGuard."
+        "Factcheck violation in llm response has been detected by AutoGuard."
     )

tests/test_configs/autoguard_factcheck/config.yml

@@ -7,9 +7,6 @@ rails:
     autoguard:
       parameters:
         fact_check_endpoint: "https://nvidia.autoalign.ai/factcheck"
-  input:
-    flows:
-      - autoguard factcheck input
   output:
     flows:
       - autoguard factcheck output