Merge pull request #78 from VinciGit00/robots_test

Robots test 🤖

Author: PeriniM

examples/benchmarks/SmartScraper/benchmark_docker.py

@@ -2,7 +2,6 @@
 Basic example of scraping pipeline using SmartScraper from text
 """
 
-import os
 from scrapegraphai.graphs import SmartScraperGraph
 from scrapegraphai.utils import prettify_exec_info
 

examples/openai/custom_graph_openai.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 from scrapegraphai.models import OpenAI
 from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
+from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode
 load_dotenv()
 
 # ************************************************
@@ -31,6 +31,12 @@
 llm_model = OpenAI(graph_config["llm"])
 
 # define the nodes for the graph
+robot_node = RobotsNode(
+    input="url",
+    output=["is_scrapable"],
+    node_config={"llm": llm_model}
+)
+
 fetch_node = FetchNode(
     input="url | local_dir",
     output=["doc"],
@@ -57,17 +63,19 @@
 
 graph = BaseGraph(
     nodes={
+        robot_node,
         fetch_node,
         parse_node,
         rag_node,
         generate_answer_node,
     },
     edges={
+        (robot_node, fetch_node),
         (fetch_node, parse_node),
         (parse_node, rag_node),
         (rag_node, generate_answer_node)
     },
-    entry_point=fetch_node
+    entry_point=robot_node
 )
 
 # ************************************************

examples/single_node/robot_node.py

@@ -0,0 +1,48 @@
+"""
+Example of custom graph using existing nodes
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.models import OpenAI
+from scrapegraphai.nodes import RobotsNode
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-3.5-turbo",
+        "temperature": 0,
+        "streaming": True
+    },
+}
+
+# ************************************************
+# Define the node
+# ************************************************
+
+llm_model = OpenAI(graph_config["llm"])
+
+robots_node = RobotsNode(
+    input="url",
+    output=["is_scrapable"],
+    node_config={"llm": llm_model}
+)
+
+# ************************************************
+# Test the node
+# ************************************************
+
+state = {
+    "url": "https://twitter.com/home"
+}
+
+result = robots_node.execute(state)
+
+print(result)

manual deployment/commit_and_push_with_tests.sh

@@ -0,0 +1,34 @@
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <commit_message>"
+    exit 1
+fi
+
+cd ..
+
+# Extract the commit message from the argument
+commit_message="$1"
+
+# Run Pylint on the specified Python files
+pylint pylint scrapegraphai/**/*.py scrapegraphai/*.py tests/**/*.py
+
+cd tests
+
+# Run pytest
+if ! pytest; then
+    echo "Pytest failed. Aborting commit and push."
+    exit 1
+fi
+
+cd ..
+
+# Make the pull
+git pull
+
+# Add the modified files to the Git repository
+git add .
+
+# Commit the changes with the provided message
+git commit -m "$commit_message"
+
+# Push the changes to the remote repository
+git push

poetry.lock

@@ -4,6 +4,7 @@
 import time
 from langchain_community.callbacks import get_openai_callback
 
+
 class BaseGraph:
     """
     BaseGraph manages the execution flow of a graph composed of interconnected nodes.
@@ -81,7 +82,6 @@ def execute(self, initial_state: dict) -> dict:
 
             with get_openai_callback() as cb:
                 result = current_node.execute(state)
-                
                 node_exec_time = time.time() - curr_time
                 total_exec_time += node_exec_time
 

scrapegraphai/graphs/base_graph.py

@@ -5,3 +5,4 @@
 from .nodes_metadata import nodes_metadata
 from .schemas import graph_schema
 from .models_tokens import models_tokens
+from .robots import robots_dictionary

scrapegraphai/helpers/__init__.py

@@ -0,0 +1,12 @@
+
+""" 
+Module for mapping the models in ai agents
+"""
+robots_dictionary = {
+    "gpt-3.5-turbo": ["GPTBot", "ChatGPT-user"],
+    "gpt-4-turbo": ["GPTBot", "ChatGPT-user"],
+    "claude": ["Claude-Web", "ClaudeBot"],
+    "perplexity": "PerplexityBot",
+    "cohere": "cohere-ai",
+    "anthropic": "anthropic-ai"
+}

scrapegraphai/helpers/robots.py

@@ -12,3 +12,4 @@
 from .image_to_text_node import ImageToTextNode
 from .search_internet_node import SearchInternetNode
 from .generate_scraper_node import GenerateScraperNode
+from .robots_node import RobotsNode

scrapegraphai/nodes/__init__.py

@@ -9,9 +9,9 @@
 from langchain_community.document_transformers import EmbeddingsRedundantFilter
 from langchain_community.embeddings import HuggingFaceHubEmbeddings
 from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OllamaEmbeddings
 from langchain_openai import OpenAIEmbeddings, AzureOpenAIEmbeddings
 from ..models import OpenAI, Ollama, AzureOpenAI, HuggingFace
-from langchain_community.embeddings import OllamaEmbeddings
 from .base_node import BaseNode
 
 
@@ -97,7 +97,7 @@ def execute(self, state):
             # remove streaming and temperature
             params.pop("streaming", None)
             params.pop("temperature", None)
-            
+
             embeddings = OllamaEmbeddings(**params)
         elif isinstance(embedding_model, HuggingFace):
             embeddings = HuggingFaceHubEmbeddings(model=embedding_model.model)