A Technical End-to-End Python Guide to Automatic Web Browsing and Results Reporting

				
					from langchain.callbacks.base import BaseCallbackHandler
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.retrievers.web_research import WebResearchRetriever
import faiss
from langchain.vectorstores import FAISS
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.docstore import InMemoryDocstore
from langchain.chat_models import ChatOpenAI
from langchain.utilities import GoogleSearchAPIWrapper
import openai
import os
import logging

os.environ["GOOGLE_API_KEY"] = "" # Get it at https://console.cloud.google.com/apis/api/customsearch.googleapis.com/credentials
os.environ["GOOGLE_CSE_ID"] = "" # Get it at https://programmablesearchengine.google.com/
os.environ["OPENAI_API_BASE"] = "https://api.openai.com/v1"
os.environ["OPENAI_API_KEY"] = "" # Get it at https://openai.com/account/api-keys
openai.api_key = "" # Get it at https://openai.com/account/api-keys

# Initialize required components

embeddings_model = OpenAIEmbeddings()
embedding_size = 1536
index = faiss.IndexFlatL2(embedding_size)
vectorstore_public = FAISS(embeddings_model.embed_query, index, InMemoryDocstore({}), {})

llm = ChatOpenAI(model_name="gpt-3.5-turbo-16k", temperature=0, streaming=True)
search = GoogleSearchAPIWrapper()

web_retriever = WebResearchRetriever.from_llm(
    vectorstore=vectorstore_public,
    llm=llm,
    search=search,
    num_search_results=3 #This should be adjusted depending on how many sources are desired
)

class PrintRetrievalHandler(BaseCallbackHandler):
    def __init__(self):
        self.results = []

    def on_retriever_start(self, query: str, **kwargs):
        self.results.append(f"**Question:** {query}\n")

    def on_retriever_end(self, documents, **kwargs):
        for idx, doc in enumerate(documents):
            source = doc.metadata.get("source", "Source not available")
            title = doc.metadata.get("title", "Title not available")
            snippet = doc.page_content[:2000]  # Using a snippet for summary

            # Generate summary for each document using the provided method
            summary_prompt = f"Please summarize the following content from '{title}' ({source}):\n\n{snippet}"
            summary = self.extract_chunk(summary_prompt)

            # Append results with source, title, URL, and summary
            self.results.append(f"**Source:** {source}\n**Title:** {title}\n**Summary:**\n{summary}\n")

    def extract_chunk(self, content):
        """
        Applies a prompt to some input content and extracts the response from OpenAI's ChatCompletion API.
        """
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo-16k", messages=[{"role": "user", "content": content}], temperature=0
        )
        return response["choices"][0]["message"]["content"]

# User input
question = "What is does the SP500 tend to do in January?"

# Instantiate the PrintRetrievalHandler
retrieval_streamer_cb = PrintRetrievalHandler()

# Perform retrieval and answer generation
qa_chain = RetrievalQAWithSourcesChain.from_chain_type(llm, retriever=web_retriever)
result = qa_chain({"question": question}, callbacks=[retrieval_streamer_cb])

# Print the answer and sources with summaries
print("\nMain Answer:\n")
print(result['answer'])

print("\nSources and Summaries:\n")
for item in retrieval_streamer_cb.results:
    print(item)