Notes

pip install chromadb
pip install langchain
pip install pypdf
pip install sentence-transformers

###################################################################################

from pypdf import PdfReader
import chromadb
from chromadb.config import Settings
from chromadb.utils import embedding_functions
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Step 1: Read the PDF file
reader = PdfReader("/content/Carrier_estimate(1).pdf")
pdf_texts = [p.extract_text().strip() for p in reader.pages]

# Step 2: Filter out empty strings
pdf_texts = [text for text in pdf_texts if text]

# Step 3: Join the text from all pages
joined_text = 'nn'.join(pdf_texts)

# Step 4: Split the text into smaller chunks
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = splitter.split_text(joined_text)

# Step 5: Initialize the ChromaDB client
client = chromadb.Client()

# Step 6: Use SentenceTransformerEmbeddingFunction from Chroma
embedding_function = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="all-MiniLM-L6-v2")

# Step 7: Create a Chroma collection (initialize the vector store)
collection = client.create_collection(name="pdf_collection2", embedding_function=embedding_function)

# Step 8: Add the documents (chunks) to Chroma collection with IDs
ids = [str(i) for i in range(len(chunks))]
collection.add(documents=chunks, ids=ids)

# Step 9: Function to query the embeddings with input length limit
def query_embeddings(query, collection, max_length=512):
# Limit the length of the query
if len(query) > max_length:
query = query[:max_length]

# Perform a similarity search in the vector store using Chroma
results = collection.query(query_texts=[query], n_results=1)

return results["documents"][0] if results["documents"] else []

# Step 10: Ask a question and get the result directly from the embeddings
query = "CONTINUED - Front Elevation?"
results = query_embeddings(query, collection)

# Step 11: Print the results
for idx, result in enumerate(results, start=1):
print(f"Result {idx}:")
print(result)
print("n")