enhance: add scrapegraph-sdk integration PR2206 (camel-ai#2259)

Author: Wendong-Fan

camel/loaders/__init__.py

@@ -20,6 +20,7 @@
 from .jina_url_reader import JinaURLReader
 from .mineru_extractor import MinerU
 from .pandas_reader import PandasReader
+from .scrapegraph_reader import ScrapeGraphAI
 from .unstructured_io import UnstructuredIO
 
 __all__ = [
@@ -34,4 +35,5 @@
     'PandasReader',
     'MinerU',
     'Crawl4AI',
+    'ScrapeGraphAI',
 ]

camel/loaders/scrapegraph_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,20 +10,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
 from typing import Any, Dict, Optional
 
-from pydantic import BaseModel
-
 
 class ScrapeGraphAI:
-    r"""ScrapeGraphAI allows you to perform AI-powered web scraping and searching.
+    r"""ScrapeGraphAI allows you to perform AI-powered web scraping and
+    searching.
 
     Args:
-        api_key (Optional[str]): API key for authenticating with the ScrapeGraphAI
-            API.
+        api_key (Optional[str]): API key for authenticating with the
+            ScrapeGraphAI API.
 
     References:
         https://scrapegraph.ai/
@@ -50,7 +49,8 @@ def search(
             user_prompt (str): The search query or instructions.
 
         Returns:
-            Dict[str, Any]: The search results including answer and reference URLs.
+            Dict[str, Any]: The search results including answer and reference
+                URLs.
 
         Raises:
             RuntimeError: If the search process fails.
@@ -72,8 +72,8 @@ def scrape(
         Args:
             website_url (str): The URL to scrape.
             user_prompt (str): Instructions for what data to extract.
-            website_html (Optional[str]): Optional HTML content to use instead of
-                fetching from the URL.
+            website_html (Optional[str]): Optional HTML content to use instead
+                of fetching from the URL.
 
         Returns:
             Dict[str, Any]: The scraped data including request ID and result.

examples/loaders/scrapegraph_example.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,10 +10,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 """
-Example demonstrating how to use the ScrapeGraphAI reader for web scraping and searching.
+Example demonstrating how to use the ScrapeGraphAI reader for web scraping and
+searching.
 
 This example shows:
 1. How to initialize the ScrapeGraphAI reader
@@ -23,38 +24,38 @@
 """
 
 import os
-from typing import Dict, Any
+from typing import Any, Dict
 
 from camel.loaders.scrapegraph_reader import ScrapeGraphAI
 
 
 def search_example(api_key: str) -> Dict[str, Any]:
-    """Example of performing an AI-powered web search."""
+    r"""Example of performing an AI-powered web search."""
     # Initialize the ScrapeGraphAI reader
     scraper = ScrapeGraphAI(api_key=api_key)
-    
+
     try:
         # Perform a search
         search_query = "What are the latest developments in AI?"
         result = scraper.search(user_prompt=search_query)
-        
+
         print("\nSearch Results:")
         print(f"Answer: {result.get('answer', 'No answer found')}")
         print("References:")
         for url in result.get('references', []):
             print(f"- {url}")
-            
+
         return result
     finally:
         # Always close the connection
         scraper.close()
 
 
 def scrape_example(api_key: str) -> Dict[str, Any]:
-    """Example of scraping a website with specific instructions."""
+    r"""Example of scraping a website with specific instructions."""
     # Initialize the ScrapeGraphAI reader
     scraper = ScrapeGraphAI(api_key=api_key)
-    
+
     try:
         # Scrape a website with specific instructions
         website_url = "https://example.com"
@@ -64,17 +65,16 @@ def scrape_example(api_key: str) -> Dict[str, Any]:
         2. All paragraph texts
         3. Any links to other pages
         """
-        
+
         result = scraper.scrape(
-            website_url=website_url,
-            user_prompt=instructions
+            website_url=website_url, user_prompt=instructions
         )
-        
+
         print("\nScraping Results:")
         print(f"Request ID: {result.get('request_id', 'No ID')}")
         print("Extracted Data:")
         print(result.get('result', {}))
-        
+
         return result
     finally:
         # Always close the connection
@@ -84,17 +84,17 @@ def scrape_example(api_key: str) -> Dict[str, Any]:
 def main():
     # Get API key from environment variable or use a placeholder
     api_key = os.environ.get("SCRAPEGRAPH_API_KEY", "your_api_key_here")
-    
+
     if api_key == "your_api_key_here":
         print("Please set your SCRAPEGRAPH_API_KEY environment variable")
         return
-    
+
     print("Running search example...")
     search_example(api_key)
-    
+
     print("\nRunning scrape example...")
     scrape_example(api_key)
 
 
 if __name__ == "__main__":
-    main() 
+    main()

pyproject.toml

@@ -240,6 +240,7 @@ owl = [
     "pandas>=1.5.3,<2",
     "rouge>=1.0.1,<2",
     "crawl4ai>=0.3.745",
+    "scrapegraph-py>=1.12.0,<2",
 ]
 all = [
     "numpy~=1.26",
@@ -352,6 +353,7 @@ all = [
     "crawl4ai>=0.3.745",
     "pyautogui>=0.9.54,<0.10",
     "pyobvector>=0.1.18",
+    "scrapegraph-py>=1.12.0,<2",
 ]
 
 [project.urls]
@@ -538,6 +540,7 @@ module = [
     "crawl4ai.*",
     "pyautogui",
     "pyobvector.*",
+    "scrapegraph_py.*",
 ]
 ignore_missing_imports = true
 

test/loaders/test_scrapegraph_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2025@ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
 from unittest.mock import MagicMock, patch
@@ -22,21 +22,23 @@
 
 @pytest.fixture
 def scrapegraph_ai():
-    with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:
+    with patch("scrapegraph_py.Client") as mock_client:
         mock_client_instance = MagicMock()
         mock_client.return_value = mock_client_instance
         yield ScrapeGraphAI(api_key="test_api_key")
 
 
 def test_init_with_api_key():
-    with patch("camel.loaders.scrapegraph_reader.Client") as mock_client:
+    with patch("scrapegraph_py.Client") as mock_client:
         ScrapeGraphAI(api_key="test_api_key")
         mock_client.assert_called_once_with(api_key="test_api_key")
 
 
 def test_init_with_env_var():
-    with patch("camel.loaders.scrapegraph_reader.Client") as mock_client, \
-         patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}):
+    with (
+        patch("scrapegraph_py.Client") as mock_client,
+        patch.dict(os.environ, {"SCRAPEGRAPH_API_KEY": "env_api_key"}),
+    ):
         ScrapeGraphAI()
         mock_client.assert_called_once_with(api_key="env_api_key")
 
@@ -47,13 +49,19 @@ def test_search_success(scrapegraph_ai):
 
     result = scrapegraph_ai.search("test query")
     assert result == mock_response
-    scrapegraph_ai.client.searchscraper.assert_called_once_with(user_prompt="test query")
+    scrapegraph_ai.client.searchscraper.assert_called_once_with(
+        user_prompt="test query"
+    )
 
 
 def test_search_failure(scrapegraph_ai):
-    scrapegraph_ai.client.searchscraper.side_effect = Exception("Search failed")
+    scrapegraph_ai.client.searchscraper.side_effect = Exception(
+        "Search failed"
+    )
 
-    with pytest.raises(RuntimeError, match="Failed to perform search: Search failed"):
+    with pytest.raises(
+        RuntimeError, match="Failed to perform search: Search failed"
+    ):
         scrapegraph_ai.search("test query")
 
 
@@ -63,13 +71,13 @@ def test_scrape_success(scrapegraph_ai):
 
     result = scrapegraph_ai.scrape(
         website_url="https://example.com",
-        user_prompt="Extract title and description"
+        user_prompt="Extract title and description",
     )
     assert result == mock_response
     scrapegraph_ai.client.smartscraper.assert_called_once_with(
         website_url="https://example.com",
         user_prompt="Extract title and description",
-        website_html=None
+        website_html=None,
     )
 
 
@@ -80,26 +88,28 @@ def test_scrape_with_html(scrapegraph_ai):
     result = scrapegraph_ai.scrape(
         website_url="https://example.com",
         user_prompt="Extract title and description",
-        website_html="<html>test</html>"
+        website_html="<html>test</html>",
     )
     assert result == mock_response
     scrapegraph_ai.client.smartscraper.assert_called_once_with(
         website_url="https://example.com",
         user_prompt="Extract title and description",
-        website_html="<html>test</html>"
+        website_html="<html>test</html>",
     )
 
 
 def test_scrape_failure(scrapegraph_ai):
     scrapegraph_ai.client.smartscraper.side_effect = Exception("Scrape failed")
 
-    with pytest.raises(RuntimeError, match="Failed to perform scrape: Scrape failed"):
+    with pytest.raises(
+        RuntimeError, match="Failed to perform scrape: Scrape failed"
+    ):
         scrapegraph_ai.scrape(
             website_url="https://example.com",
-            user_prompt="Extract title and description"
+            user_prompt="Extract title and description",
         )
 
 
 def test_close(scrapegraph_ai):
     scrapegraph_ai.close()
-    scrapegraph_ai.client.close.assert_called_once() 
+    scrapegraph_ai.client.close.assert_called_once()