Notes

import os
import json
import logging
from fastapi import FastAPI, HTTPException, Depends
from dotenv import load_dotenv
from fuzzywuzzy import process
from firecrawl import FirecrawlApp
from langchain_groq import ChatGroq
from pydantic import BaseModel
from typing import Dict
from models import AllProperties, ExtractPropertyDetails, QueryModel, PropertyInfoRequest

# Load environment variables
load_dotenv()
FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY")
GROQ_API_KEY = os.getenv("GROQ_API_KEY")

# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s",
handlers=[
logging.FileHandler("app.log"), # Log to a file
logging.StreamHandler() # Log to console
]
)
logger = logging.getLogger(__name__)

# Initialize Firecrawl and FastAPI
scrapper_app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
app = FastAPI()

# Initialize LLM
llm = ChatGroq(groq_api_key=GROQ_API_KEY, model_name="llama-3.3-70b-versatile")

# Load property types from JSON file
try:
with open("all_websites.json", "r") as f:
data = json.load(f)["loopnet"]
except (FileNotFoundError, json.JSONDecodeError) as e:
logger.error("Error loading all_websites.json: %s", str(e))
raise RuntimeError("Failed to load property types from JSON.")


property_types = {prop: "for-lease" for prop in data["property_type"]["for lease"]} or {prop: "for-sale" for prop in data["property_type"]["for sale"]}


def extract_entities(user_query: str):
"""Extracts location, property type, and sale type from user query using LLM."""
prompt = f"""
Extract the following details from the user's request:
- Property type (from given list: {list(property_types.keys())})
- Location (City, State, or Country)
Instructions for Location:
-- if you got any city for example `San Francisco then location will be san-francisco-ca` explanation "San Francisco, CA" refers to San Francisco, a major city in the state of California (CA), USA
- Sale type (either 'for-sale' or 'for-lease')

User request: {user_query}

Output format (JSON): {{ "property_type": "", "location": "", "sale_type": "" }}
Instructions:
-- Do not provide extra text rather than json file
"""
try:
response = llm.invoke(prompt)
extracted_data = json.loads(response.content)
logger.info("Extracted entities: %s", extracted_data)
return extracted_data
except Exception as e:
logger.error("Error extracting entities: %s", str(e))
raise HTTPException(status_code=500, detail="Failed to extract property details.")


def generate_url(property_type: str, location: str, sale_type: str):
"""Constructs the LoopNet URL."""
return f"https://www.loopnet.com/search/{property_type}/{location}/{sale_type}/*"


@app.post("/extract-all-properties/")
async def extract_all_properties_endpoint(query_data: QueryModel):
"""API endpoint to generate LoopNet URL."""
user_query = query_data.query
try:
extracted = extract_entities(user_query)
matched_property_type = process.extractOne(extracted["property_type"], property_types.keys())[0]
url = generate_url(matched_property_type, extracted["location"], extracted["sale_type"])
print(f">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>{url}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
data = scrapper_app.extract([
url
], {
'prompt': 'Extract all available details for each property listed, including address, price, square footage, property type, description, and contact information. Ensure that each property has a link with the class name `left-h4 link`.',
'schema': AllProperties.model_json_schema(),
})
logger.info("Successfully extracted property details.")
return {"data": data}
except Exception as e:
logger.error("Error extracting all properties: %s", str(e))
raise HTTPException(status_code=500, detail="Failed to extract property listings.")


@app.post("/extract-property-info")
async def extract_property_info(request: PropertyInfoRequest):
"""API endpoint to extract details of a specific property from a given URL."""
try:
data = scrapper_app.extract([
request.url
], {
'prompt': 'Extract all available details from the property listing, including address, price, square footage, Property Type: Whether the property is for sale or rent,Flat Type: The type of flat or unit (e.g. studio, 1-bedroom, 2-bedroom, etc.), Year Built (The year the property was constructed), Description, and broker contact information.',
'schema': ExtractPropertyDetails.model_json_schema(),
})
logger.info("Successfully extracted property info for URL: %s", request.url)
return {"data": data}
except Exception as e:
logger.error("Error extracting property details: %s", str(e))
raise HTTPException(status_code=500, detail="Failed to extract property details.")

Above the code are the output of this API (extract all properties):-

"properties": [
{
"link": "https://www.loopnet.com/Listing/61-67-West-St-Brooklyn-NY/30360546/",
"price": "",
"address": "61-67 West St, Brooklyn, NY",
"Flat_Type": "",
"Year_Built": "1931",
"description": "Discover historic charm inside a fully converted factory building within a block of the East River, offering immediately available commercial space.",
"property_type": "Office, Retail",
"square_footage": "465 - 26,640 SF",
"contact_information": ""
},

In the Output "Link" key is there , each unique link have the contents like (Highlights, All available spaces , Property Overview) in the provided url

I need to extract those contents with each particular link

Write a modified prompt in the extract-property-info API in the above code