Knowledge Graph RAG
For Intelligent Data Analysis

def search_with_google_api(query):
    url = f"https://www.googleapis.com/customsearch/v1?q={query}&key={google_api}&cx={search_engine_id}"
    response = requests.get(url)
    if response.status_code == 200:
        return response.json().get("items", [])
    else:
        print(f"Error: {response.status_code}, {response.text}")
        return []

def generate_search_queries(user_input):
    """
    Generates a list of 5-7 detailed and relevant search queries for financial sentiment analysis
    based on the user's input, such as a target sector, field, or region.
    """
    prompt = f"""
You are a financial analyst and search query expert. Based on the following user input, generate a list of 5-7 search queries
for financial sentiment analysis based on user input. Ensure the queries cover diverse aspects of the topic, including sector-specific trends,
regional financial overviews, and broader financial landscapes. The queries should focus on extracting data relevant to sentiment
and performance analysis.

User Input: {user_input}

Strictly output the queries as a python list of strings. Do not add any additional comments.
"""

    response = openai.chat.completions.create(
        model="gpt-4",
        messages=[
            {"role": "system", "content": "You are an expert in generating search queries for financial sentiment analysis."},
            {"role": "user", "content": prompt}
        ],
        max_tokens=200
    )

    # Extract and clean up the list of queries
    queries =  response.choices[0].message.content.strip()
    return eval(queries)

def fetch_full_content(url):
    """
    Fetches the full content of a webpage given its URL.
    """
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"
        )
    }
    try:
        response = requests.get(url, headers=headers, timeout=10)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, "html.parser")
            paragraphs = soup.find_all("p")
            full_text = "\n".join([p.get_text() for p in paragraphs])
            return full_text.strip() if full_text else None
        else:
            print(f"Error: Unable to fetch content from {url} (Status Code: {response.status_code})")
            return None
    except Exception as e:
        print(f"Error fetching content from {url}: {e}")
        return None

def create_dataset_from_queries(queries, directory="dataset"):
    """
    Process search queries and save results as text files in the same directory.
    """
    if not os.path.exists(directory):
        os.makedirs(directory)

    file_count = 1  # To ensure unique filenames across all queries

    for query in queries:
        print(f"Processing query: {query}")
        valid_count = 0
        page_number = 1

        while valid_count < 10:
            print(f"Fetching search results, page {page_number}...")
            results = search_with_google_api(query + f"&start={page_number * 10}")

            if not results:
                print("No more results found. Try refining the query.")
                break

            for result in results:
                if valid_count >= 10:
                    break  # Stop when 10 valid documents are saved

                title = result["title"]
                link = result["link"]
                snippet = result.get("snippet", "No snippet")

                # Fetch full content of the link
                full_content = fetch_full_content(link)
                if full_content:  # Save only if content is valid
                    filename = f"{directory}/doc_{file_count}.txt"
                    with open(filename, "w", encoding="utf-8") as f:
                        f.write(f"Query: {query}\n")
                        f.write(f"Title: {title}\n")
                        f.write(f"Link: {link}\n")
                        f.write(f"Snippet: {snippet}\n\n")
                        f.write(f"Full Content:\n{full_content}")
                    print(f"Saved: {filename}")
                    valid_count += 1
                    file_count += 1
                else:
                    print(f"Skipped: {link} (No valid content)")

            page_number += 1  # Move to the next page of results

    print(f"Finished processing all queries. Total files saved: {file_count - 1}")

dataset/
├── doc_1.txt
├── doc_2.txt
├── doc_3.txt
└── ...

# User provides financial analysis topic
user_input = "Financial sentiment analysis for the electric vehicle sector in the US"

# Step 1: Generate targeted search queries using AI
queries = generate_search_queries(user_input)
print("Generated Queries:")
for i, query in enumerate(queries, 1):
    print(f"  {i}. {query}")

# Step 2: Execute complete dataset creation pipeline
create_dataset_from_queries(queries)

Generated Queries:
  1. "electric vehicle market trends US 2024 financial analysis"
  2. "EV sector stock performance US investor sentiment"
  3. "electric vehicle company earnings reports US market"
  4. "US EV industry government policies and financial impact"
  5. "electric vehicle consumer adoption rates US financial outlook"
  6. "EV battery technology investments US market sentiment"
  7. "electric vehicle charging infrastructure financial analysis US"

Processing query: electric vehicle market trends US 2024 financial analysis
Fetching search results, page 1...
Saved: dataset/doc_1.txt
Saved: dataset/doc_2.txt
...
Finished processing all queries. Total files saved: 68

dataset/
├── doc_1.txt
├── doc_2.txt
├── doc_3.txt
├── ...
└── doc_68.txt

Each file contains:
Query: [original search query]
Title: [webpage title]
Link: [source URL]
Snippet: [search result summary]
Full Content: [complete extracted text]

# Initialize in-memory graph store
graph_store = SimplePropertyGraphStore()

# Load documents from dataset directory
documents = SimpleDirectoryReader("dataset").load_data()

# Create Property Graph Index with entity extraction
index = PropertyGraphIndex.from_documents(
    documents,
    embed_model=OpenAIEmbedding(model_name="text-embedding-3-small"),
    kg_extractors=[
        SchemaLLMPathExtractor(
            llm=OpenAI(model="gpt-3.5-turbo", temperature=0.0)
        )
    ],
    property_graph_store=graph_store,  # Uses in-memory store
    show_progress=True,
    use_async=True
)

print("✅ Knowledge Graph created successfully with SimplePropertyGraphStore!")
print("✅ No external database required!")

Loading documents: 100%|██████████| 68/68 [00:02<00:00, 25.43it/s]
Extracting entities: 100%|██████████| 68/68 [01:15<00:00, 1.11s/it]
Creating graph structure: 100%|██████████| 68/68 [00:45<00:00, 1.51it/s]
✅ Knowledge Graph created successfully with SimplePropertyGraphStore!
✅ No external database required!

# Persist the knowledge graph to storage directory
index.storage_context.persist(persist_dir="./storage")

# Load index from storage in future sessions
from llama_index.core import StorageContext, load_index_from_storage

index = load_index_from_storage(
    StorageContext.from_defaults(persist_dir="./storage")
)

# Alternative: Load from existing graph store
index = PropertyGraphIndex.from_existing(
    property_graph_store=graph_store
)

storage/
├── graph_store.json
├── index_store.json
├── docstore.json
├── vector_store.json
└── image__vector_store.json

# Define retriever with graph-only mode
retriever = index.as_retriever(
    include_text=False,  # Default is true
)

# Execute retrieval query
results = retriever.retrieve("What is the summary of the financial texts?")

# Display retrieved results
for record in results:
    print(record.text)

The financial texts analyze the electric vehicle sector in the US, focusing on:
- Market growth projections showing 35% CAGR through 2028
- Key players: Tesla, Ford, GM, and emerging startups like Rivian
- Investment trends: $120B committed to EV infrastructure
- Regulatory impacts: Federal tax incentives and state-level mandates
- Consumer adoption: Current 7% market share projected to reach 25% by 2030
- Supply chain challenges: Battery material shortages and manufacturing scaling

"Show me companies investing in battery technology"
"What are the main challenges facing EV startups?"
"Compare market positions of Tesla and traditional automakers"
"Analyze the impact of government policies on EV adoption"
"Identify key growth drivers in the electric vehicle sector"