from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from sqlalchemy import and_, or_, func, text
from typing import List, Optional, Dict, Any
from pydantic import BaseModel, Field
import re
import json
import os
from datetime import datetime, date
from langchain_google_genai import ChatGoogleGenerativeAI
from dotenv import load_dotenv
from pathlib import Path

from db.db_connection import get_db
from db.models.house_model import House, HouseResponse

# Load environment variables
BASE_DIR = Path(__file__).resolve().parent.parent.parent
env_path = BASE_DIR / ".env.local"
if env_path.exists():
    load_dotenv(dotenv_path=env_path)
else:
    print(f"Warning: .env.local file not found at {env_path}")
    print("Please copy .env.template to .env.local and add your GEMINI_API_KEY")

# Initialize Gemini
api_key = os.getenv("GEMINI_API_KEY")
if not api_key:
    print("Warning: GEMINI_API_KEY not set. Using rule-based fallback.")
    llm = None
else:
    try:
        llm = ChatGoogleGenerativeAI(
            model="gemini-2.0-flash",
            google_api_key=api_key,
            temperature=0.3
        )
    except Exception as e:
        print(f"Warning: Failed to initialize Gemini: {e}. Using rule-based fallback.")
        llm = None

router = APIRouter(
    prefix="/chatbot",
    tags=["Property Chatbot AI"]
)

# ===============================
# Pydantic Models
# ===============================
class ChatbotQuery(BaseModel):
    user_message: str = Field(..., example="I want a 3 bedroom condo under RM 2000 in KL")
    conversation_id: Optional[str] = None
    user_preferences: Optional[Dict[str, Any]] = None

class ChatbotResponse(BaseModel):
    message: str
    properties: List[HouseResponse]
    conversation_id: str
    filters_applied: Dict[str, Any]
    total_found: int
    suggestions: List[str]

# ===============================
# Natural Language Processing Functions
# ===============================
def extract_price_range(text: str) -> tuple[Optional[float], Optional[float]]:
    """Extract price range from user message"""
    text = text.lower()
    
    # Remove commas and normalize currency
    text = re.sub(r'rm\s*', '', text)
    text = re.sub(r'ringgit\s*', '', text)
    text = re.sub(r',', '', text)
    
    # Price patterns
    patterns = [
        r'under\s+(\d+(?:\.\d+)?k?)',  # under 2000, under 2k
        r'below\s+(\d+(?:\.\d+)?k?)',  # below 1500
        r'less\s+than\s+(\d+(?:\.\d+)?k?)',  # less than 2000
        r'maximum\s+(\d+(?:\.\d+)?k?)',  # maximum 2500
        r'max\s+(\d+(?:\.\d+)?k?)',  # max 3000
        r'budget\s+(?:of\s+|is\s+)?(\d+(?:\.\d+)?k?)',  # budget 2000, budget of 2k
        r'around\s+(\d+(?:\.\d+)?k?)',  # around 2000
        r'(\d+(?:\.\d+)?k?)\s*(?:to|[-–])\s*(\d+(?:\.\d+)?k?)',  # 1500-2000, 1.5k to 2k
        r'between\s+(\d+(?:\.\d+)?k?)\s*(?:and|to|[-–])\s*(\d+(?:\.\d+)?k?)',  # between 1000 and 2000
    ]
    
    def convert_to_number(value_str: str) -> float:
        """Convert string like '2k' or '2000' to float"""
        if value_str.endswith('k'):
            return float(value_str[:-1]) * 1000
        return float(value_str)
    
    for pattern in patterns:
        match = re.search(pattern, text)
        if match:
            if len(match.groups()) == 1:  # Single price (upper limit)
                max_price = convert_to_number(match.group(1))
                return (None, max_price)
            elif len(match.groups()) == 2:  # Price range
                min_price = convert_to_number(match.group(1))
                max_price = convert_to_number(match.group(2))
                return (min_price, max_price)
    
    return (None, None)

def extract_bedrooms(text: str) -> Optional[int]:
    """Extract number of bedrooms from user message"""
    text = text.lower()
    
    patterns = [
        r'(\d+)\s*(?:bedroom|bed|br)',
        r'(\d+)\s*bed',
        r'(\d+)br',
        r'(\d+)\s*room',
    ]
    
    for pattern in patterns:
        match = re.search(pattern, text)
        if match:
            return int(match.group(1))
    
    return None

def extract_property_type(text: str) -> Optional[str]:
    """Extract property type from user message"""
    text = text.lower()
    
    type_mapping = {
        'condo': 'Condominium',
        'condominium': 'Condominium',
        'apartment': 'Apartment',
        'house': 'Landed House',
        'landed': 'Landed House',
        'terrace': 'Terrace',
        'semi-d': 'Semi-Detached',
        'semi detached': 'Semi-Detached',
        'bungalow': 'Bungalow',
        'townhouse': 'Townhouse',
        'studio': 'Studio',
        'flat': 'Flat'
    }
    
    for key, value in type_mapping.items():
        if key in text:
            return value
    
    return None

def extract_location(text: str) -> List[str]:
    """Extract location keywords from user message"""
    text = text.lower()
    
    # Common Sabah locations
    locations = [
        'kota kinabalu', 'kk', 'sabah',
        'penampang', 'tuaran', 'sandakan', 'tawau', 'lahad datu',
        'keningau', 'semporna', 'kudat', 'beaufort', 'sipitang',
        'kota belud', 'ranau', 'kota marudu', 'kunak', 'tongod',
        'beluran', 'kinabatangan', 'telupid', 'pitas', 'putatan',
        'likas', 'lintas', 'kolombong', 'inanam', 'menggatal'
    ]
    
    found_locations = []
    for location in locations:
        if location in text:
            found_locations.append(location)
    
    # Special handling for workplace locations
    # Extract location from workplace names like "Menara Kinabalu Likas"
    workplace_patterns = [
        r'menara\s+kinabalu\s+likas',
        r'likas\s+menara',
        r'kinabalu\s+likas',
        r'likas\s+kinabalu'
    ]
    
    for pattern in workplace_patterns:
        if re.search(pattern, text):
            if 'likas' not in found_locations:
                found_locations.append('likas')
            break
    
    return found_locations

def detect_non_sabah_location(text: str) -> Optional[str]:
    """Detect if user is requesting properties outside Sabah"""
    text = text.lower()
    
    # Non-Sabah locations (Peninsular Malaysia and other states)
    non_sabah_locations = [
        'kuala lumpur', 'kl', 'selangor', 'johor', 'johor bahru', 'jb',
        'penang', 'georgetown', 'perak', 'kedah', 'kelantan', 'terengganu',
        'pahang', 'negeri sembilan', 'ns', 'melaka', 'malacca', 'perlis',
        'putrajaya', 'cyberjaya', 'shah alam', 'petaling jaya', 'pj',
        'klang', 'subang', 'cheras', 'ampang', 'kepong', 'wangsa maju',
        'labuan', 'langkawi', 'ipoh', 'taiping', 'alor setar', 'kangar',
        'kota bharu', 'kuala terengganu', 'kuantan', 'seremban', 'melaka',
        'johor bahru', 'pasir gudang', 'skudai', 'tampoi', 'gelang patah'
    ]
    
    for location in non_sabah_locations:
        if location in text:
            return location
    
    return None

def detect_gibberish(text: str) -> bool:
    """Detect if the user message is gibberish or nonsensical"""
    text = text.lower().strip()
    
    # Check for very short messages that are likely gibberish
    if len(text) < 3:
        return True
    
    # Check for repeated characters (like "aaaa", "qqqq")
    if len(set(text)) <= 2 and len(text) > 3:
        return True
    
    # Check for random character sequences
    gibberish_patterns = [
        r'^[a-z]{1,2}[a-z]{1,2}[a-z]{1,2}[a-z]{1,2}[a-z]{1,2}$',  # Random 5+ letter sequences
        r'^[qwertyuiopasdfghjklzxcvbnm]{5,}$',  # Random keyboard mashing
        r'^[a-z]{1,3}[a-z]{1,3}[a-z]{1,3}[a-z]{1,3}$',  # Repeated patterns
    ]
    
    for pattern in gibberish_patterns:
        if re.match(pattern, text):
            return True
    
    # Check for common gibberish words/phrases
    gibberish_words = [
        'laskdjasdjqwe', 'asdf', 'qwerty', 'zxcvbn', 'hjkl',
        'fghjkl', 'qwertyuiop', 'asdfghjkl', 'zxcvbnm',
        'test', 'testing', 'abc', 'xyz', '123', 'hello world',
        'random', 'gibberish', 'nonsense', 'blah', 'bleh'
    ]
    
    if text in gibberish_words:
        return True
    
    # Check if message has no meaningful words (no vowels in sequence)
    if not re.search(r'[aeiou]{1,2}', text):
        return True
    
    # Check for excessive special characters or numbers mixed with letters randomly
    if re.search(r'[a-z]{1,2}\d{1,2}[a-z]{1,2}\d{1,2}[a-z]{1,2}', text):
        return True
    
    return False

def generate_gibberish_response(user_message: str) -> str:
    """Generate response for gibberish messages"""
    return """I'm not sure what you're looking for! 😅

It seems like your message might have been typed accidentally or contains some random characters.

I'm here to help you find the perfect property in Sabah! Please try asking me something like:

🏠 **"I need a 2 bedroom house in Kota Kinabalu"**
💰 **"Show me properties under RM 2000"**
📍 **"Find apartments near Penampang"**
👨‍👩‍👧‍👦 **"I have a family of 4, what do you recommend?"**

Just let me know what you're looking for and I'll help you find the best Sabah properties! 😊"""

def generate_non_sabah_location_response(user_message: str, detected_location: str) -> str:
    """Generate response when user requests properties outside Sabah"""
    
    # Suggest Sabah alternatives based on the requested location
    sabah_alternatives = {
        'kuala lumpur': 'Kota Kinabalu',
        'kl': 'Kota Kinabalu', 
        'selangor': 'Kota Kinabalu',
        'johor': 'Sandakan or Tawau',
        'penang': 'Kota Kinabalu',
        'perak': 'Kota Kinabalu',
        'kedah': 'Kota Kinabalu',
        'kelantan': 'Sandakan',
        'terengganu': 'Sandakan',
        'pahang': 'Kota Kinabalu',
        'negeri sembilan': 'Kota Kinabalu',
        'melaka': 'Kota Kinabalu',
        'perlis': 'Kota Kinabalu',
        'putrajaya': 'Kota Kinabalu',
        'cyberjaya': 'Kota Kinabalu'
    }
    
    suggested_location = sabah_alternatives.get(detected_location.lower(), 'Kota Kinabalu')
    
    return f"""I understand you're looking for properties in {detected_location.title()}, but KitaMove specializes exclusively in Sabah properties! 🏔️

Since we focus on Sabah rentals, I'd be happy to help you find similar properties in **{suggested_location}** instead, which offers:

✅ **Urban convenience** like {detected_location.title()}
✅ **Modern amenities** and facilities  
✅ **Great connectivity** and transportation
✅ **Diverse property options** from budget to luxury

Would you like me to search for properties in **{suggested_location}**? Just let me know:
🏠 **How many bedrooms do you need?**
💰 **What's your monthly budget?**

I can also show you other great Sabah locations like Penampang, Tuaran, or Sandakan! 😊"""

def extract_furnishing(text: str) -> Optional[str]:
    """Extract furnishing status from user message"""
    text = text.lower()
    
    if any(word in text for word in ['fully furnished', 'full furnished', 'furnished']):
        return 'Fully furnished'
    elif any(word in text for word in ['partially furnished', 'partial furnished', 'semi furnished', 'semi-furnished', 'partly furnished']):
        return 'Partially furnished'
    elif any(word in text for word in ['unfurnished', 'empty', 'not furnished']):
        return 'Unfurnished'
    
    return None

def extract_amenities(text: str) -> List[str]:
    """Extract desired amenities from user message"""
    text = text.lower()
    
    amenity_keywords = {
        'pool': 'Swimming Pool',
        'swimming': 'Swimming Pool',
        'gym': 'Gymnasium',
        'fitness': 'Gymnasium',
        'parking': 'Parking',
        'security': '24hr Security',
        'playground': 'Playground',
        'bbq': 'BBQ Area',
        'garden': 'Garden',
        'balcony': 'Balcony',
        'lift': 'Lift',
        'elevator': 'Lift'
    }
    
    found_amenities = []
    for keyword, amenity in amenity_keywords.items():
        if keyword in text:
            found_amenities.append(amenity)
    
    return found_amenities

# ===============================
# Gemini AI Functions
# ===============================
def extract_parameters_with_gemini(user_message: str) -> Dict[str, Any]:
    """Use Gemini AI to extract search parameters from natural language"""
    if not llm:
        # Fallback to rule-based extraction if Gemini is not available
        return {
            "min_price": extract_price_range(user_message)[0],
            "max_price": extract_price_range(user_message)[1],
            "bedrooms": extract_bedrooms(user_message),
            "property_type": extract_property_type(user_message),
            "locations": extract_location(user_message),
            "furnishing": extract_furnishing(user_message),
            "amenities": extract_amenities(user_message)
        }
    
    try:
        prompt = f"""
You are a property search assistant for Sabah, Malaysia. Extract search parameters from this user message: "{user_message}"

Return a JSON object with these fields (use null for missing info):
{{
    "min_price": number or null,
    "max_price": number or null,  
    "bedrooms": number or null,
    "property_type": "Condominium" | "Apartment" | "Landed House" | "Terrace" | "Semi-Detached" | "Bungalow" | "Townhouse" | "Studio" | "Flat" | null,
    "locations": ["location1", "location2"] or [],
    "furnishing": "Fully furnished" | "Partially furnished" | "Unfurnished" | null,
    "amenities": ["Swimming Pool", "Gymnasium", "Parking", "24hr Security", "Playground", "BBQ Area", "Garden", "Balcony", "Lift"] or [],
    "exclude_property_ids": [1, 2, 3] or [],
    "prefer_different_areas": true or false
}}

Important extraction rules:
- Service scope: KitaMove specializes EXCLUSIVELY in Sabah properties. If user requests properties outside Sabah (Kuala Lumpur, KL, Selangor, Johor, Penang, etc.), this should be handled by the non-Sabah location detection system, not by parameter extraction.
- Budget interpretation (works with or without RM prefix):
  * "My budget is more than RM 2,500" = set min_price to 2500
  * "My budget is less than RM 2,500" = set max_price to 2500
  * "Budget around RM X" = set max_price to X
  * "Budget around 1900" = set max_price to 1900 (without RM)
  * "Budget is 2000/month" = set max_price to 2000 (without RM)
  * "I need a house for 2 people budget around 1900/month" = set max_price to 1900
  * "more affordable properties within RM 2,500 and below" = set max_price to 2500
  * "affordable properties within RM 2,500 and below" = set max_price to 2500
  * "more property options" + "within RM 2,500" = set max_price to 2500
  * "My budget is more than RM 2,500" = set min_price to 2500 (user wants properties above this price)
  * "My budget is less than RM 2,500" = set max_price to 2500 (user wants properties below this price)
  * "under 1500", "below 2000", "less than 3000" = set max_price accordingly (without RM)
  * "around 1800", "budget 2500", "maximum 3500" = set max_price accordingly (without RM)
- Luxury property interpretation:
  * "luxury", "high-end", "premium", "expensive", "most expensive", "top-tier", "upscale", "exclusive", "deluxe", "lavish", "opulent", "grand", "sophisticated", "elite", "prestigious", "posh", "fancy", "extravagant", "high-class", "upper-class", "wealthy", "rich", "costly", "pricey", "high-priced", "top-end", "best", "finest", "superior", "exceptional", "outstanding" = set min_price to 5000 and max_price to 20000 (luxury range)
  * "luxury house", "luxury condo", "luxury apartment", "premium property", "expensive house", "high-end house", "deluxe property" = set min_price to 5000 and max_price to 20000
  * "I want the most expensive", "show me expensive properties", "high-end properties", "premium properties", "luxury properties" = set min_price to 5000 and max_price to 20000
  * "cheaper option", "affordable", "budget-friendly", "low-cost", "economical", "inexpensive", "budget" = set max_price to 2000
  * "luxury house rental", "expensive rental", "premium rental" = set min_price to 5000 and max_price to 20000
- Family size interpretation:
  * "I have a big family" = set bedrooms to 3 or 4 (prefer 3)
  * "I have a small family" = set bedrooms to 1 or 2 (prefer 2)
- Location interpretation:
  * "near city centre" or "city centre" = add "Kota Kinabalu" to locations
  * "preferably in different areas like X, Y, Z" = add X, Y, Z to locations
  * "in different areas" or "different locations" = set prefer_different_areas to true
  * Recognize: "KK", "Kota Kinabalu", "Penampang", "Tuaran", "Sandakan", "Tawau", "Lahad Datu", "Keningau", "Semporna", "Kudat", "Likas", "Lintas", "Kolombong", "Inanam", "Menggatal"
  * Workplace location extraction:
    * "near my workplace in Menara Kinabalu Likas" = add "Likas" to locations
    * "workplace in Kota Kinabalu" = add "Kota Kinabalu" to locations
    * "office in Penampang" = add "Penampang" to locations
    * Extract location from workplace names: "Menara Kinabalu Likas" → "Likas", "Wisma Merdeka Kota Kinabalu" → "Kota Kinabalu"
- Exclusion interpretation:
  * "exclude property IDs: 1, 2, 3" = set exclude_property_ids to [1, 2, 3]
  * "different property options" or "NEW areas" = set prefer_different_areas to true
  * "more options" or "more properties" = keep prefer_different_areas false (show more from same area first)
  * "show me more" or "get more" = keep prefer_different_areas false (show more from same area first)
- Property types: condo/condominium, apartment, house/landed, terrace, semi-d/semi-detached, bungalow, townhouse, studio, flat
- Furnishing: "fully furnished" = "Fully furnished", "partially furnished"/"semi furnished" = "Partially furnished", "unfurnished" = "Unfurnished"
- Only return the JSON object, no explanation.
"""
        
        response = llm.invoke(prompt)
        
        # Parse the JSON response (handle markdown-wrapped JSON)
        response_content = response.content.strip()
        
        # Remove markdown code block wrapper if present
        if response_content.startswith('```json'):
            response_content = response_content[7:]  # Remove ```json
        if response_content.startswith('```'):
            response_content = response_content[3:]   # Remove ```
        if response_content.endswith('```'):
            response_content = response_content[:-3]  # Remove ```
        
        response_content = response_content.strip()
        
        result = json.loads(response_content)
        
        # Validate and clean the response
        validated_result = {
            "min_price": result.get("min_price"),
            "max_price": result.get("max_price"),
            "bedrooms": result.get("bedrooms"),
            "property_type": result.get("property_type"),
            "locations": result.get("locations", []),
            "furnishing": result.get("furnishing"),
            "amenities": result.get("amenities", []),
            "exclude_property_ids": result.get("exclude_property_ids", []),
            "prefer_different_areas": result.get("prefer_different_areas", False)
        }
        
        return validated_result
        
    except Exception as e:
        print(f"Gemini extraction failed: {e}, falling back to rule-based")
        # Fallback to rule-based extraction
        return {
            "min_price": extract_price_range(user_message)[0],
            "max_price": extract_price_range(user_message)[1],
            "bedrooms": extract_bedrooms(user_message),
            "property_type": extract_property_type(user_message),
            "locations": extract_location(user_message),
            "furnishing": extract_furnishing(user_message),
            "amenities": extract_amenities(user_message),
            "exclude_property_ids": [],
            "prefer_different_areas": False
        }

def analyze_price_positioning(properties: List[House], filters_applied: Dict[str, Any], db: Session) -> Dict[str, Any]:
    """
    Analyze if the returned properties represent the cheapest available options
    """
    if not properties:
        return {"is_cheapest": False, "price_analysis": "no_properties"}
    
    try:
        # Get the highest price from our returned properties
        max_returned_price = max(float(prop.price) for prop in properties)
        min_returned_price = min(float(prop.price) for prop in properties)
        
        # Build a query to check if there are cheaper properties available
        # excluding our current max_price filter but keeping other filters
        check_query = db.query(House)
        
        # Apply all filters except max_price
        if filters_applied.get('min_price'):
            check_query = check_query.filter(House.price >= filters_applied['min_price'])
            
        if filters_applied.get('bedrooms'):
            check_query = check_query.filter(House.bedroom >= filters_applied['bedrooms'])
            
        if filters_applied.get('property_type'):
            check_query = check_query.filter(House.property_type.ilike(f'%{filters_applied["property_type"]}%'))
            
        if filters_applied.get('locations'):
            location_conditions = []
            for location in filters_applied['locations']:
                location_conditions.append(House.address.ilike(f'%{location}%'))
            check_query = check_query.filter(or_(*location_conditions))
            
        if filters_applied.get('furnishing'):
            if filters_applied['furnishing'] == 'Partially furnished':
                check_query = check_query.filter(or_(
                    House.furnishing_status == 'Partially furnished',
                    House.furnishing_status == 'Partially funished'
                ))
            else:
                check_query = check_query.filter(House.furnishing_status == filters_applied['furnishing'])
                
        if filters_applied.get('amenities'):
            for amenity in filters_applied['amenities']:
                check_query = check_query.filter(House.amenities.op('?')(amenity))
        
        # Check if there are any properties cheaper than our highest returned price
        cheaper_properties = check_query.filter(House.price < max_returned_price).count()
        
        # Check total properties matching the criteria (without price limit)
        total_matching = check_query.count()
        
        # Determine if we're showing the absolute cheapest
        is_showing_cheapest = (cheaper_properties == 0)
        
        return {
            "is_cheapest": is_showing_cheapest,
            "min_price": min_returned_price,
            "max_price": max_returned_price,
            "total_matching": total_matching,
            "cheaper_available": cheaper_properties,
            "price_analysis": "cheapest" if is_showing_cheapest else "not_cheapest"
        }
        
    except Exception as e:
        print(f"Error analyzing price positioning: {e}")
        return {"is_cheapest": False, "price_analysis": "error"}

def generate_response_with_gemini(user_message: str, properties: List[House], filters_applied: Dict[str, Any], total_found: int, db: Session = None, is_luxury_request: bool = False) -> str:
    """Use Gemini to generate more natural and contextual responses"""
    if not llm:
        # Fallback to rule-based response generation
        return generate_chatbot_response(user_message, properties, filters_applied, total_found, db)
    
    try:
        # Analyze price positioning if db session is available
        price_analysis = {"is_cheapest": False, "price_analysis": "unknown"}
        if db and properties:
            price_analysis = analyze_price_positioning(properties, filters_applied, db)
        
        # Determine how many properties to show and appropriate terminology
        num_shown = len(properties)
        if num_shown == 0:
            top_text = "No properties found"
        elif num_shown == 1:
            if price_analysis["is_cheapest"]:
                top_text = "Cheapest available option"
            else:
                top_text = "Top pick"
        elif num_shown == 2:
            if price_analysis["is_cheapest"]:
                top_text = "2 cheapest available options"
            else:
                top_text = "Top 2 picks"
        else:
            if price_analysis["is_cheapest"]:
                top_text = f"{min(num_shown, 3)} cheapest available options"
            else:
                top_text = f"Top {min(num_shown, 3)} picks"
        
        # Prepare property summaries for Gemini
        property_summaries = []
        for i, prop in enumerate(properties[:3], 1):
            property_summaries.append(f"{i}. {prop.title} - RM {prop.price}/month, {prop.bedroom} bed, {prop.bathroom} bath, {prop.property_type} in {prop.address}")
        
        properties_text = "\n".join(property_summaries) if property_summaries else "No properties found"
        
        filters_text = ", ".join([f"{k}: {v}" for k, v in filters_applied.items() if v is not None and v != []])
        
        # Add price analysis context to the prompt
        price_context = ""
        if is_luxury_request and properties:
            if num_shown == 1:
                price_context = "IMPORTANT: This is one of the most expensive luxury properties available. Emphasize the premium, high-end nature of this property."
            else:
                price_context = f"IMPORTANT: These are the {num_shown} most expensive luxury properties available. Emphasize these are premium, high-end options with top-tier amenities."
        elif price_analysis["is_cheapest"] and properties:
            if num_shown == 1:
                price_context = "IMPORTANT: This is the cheapest property available matching the user's criteria. Mention this is the most affordable option."
            else:
                price_context = f"IMPORTANT: These are the {num_shown} cheapest properties available matching the user's criteria. Emphasize these are the most affordable options."
        
        prompt = f"""
You are a friendly property assistant for Sabah, Malaysia. A user searched for: "{user_message}"

Search Results:
- Total found: {total_found}
- Showing: {num_shown} properties ({top_text})
- Applied filters: {filters_text}
- Properties shown:
{properties_text}

{price_context}

Generate a helpful, conversational response that:
1. Acknowledges their search using natural language:
   - For budget queries ("My budget is more than RM 2,500"): "Based on your budget of over RM 2,500..."
   - For family queries ("I have a big family"): "Perfect! For a big family, I found..."
   - For location queries ("near city centre"): "Great choice for city convenience..."
   - For affordability requests ("more affordable properties within RM 2,500"): "Here are more affordable options within your RM 2,500 budget..."
   - For luxury requests ("luxury properties", "expensive", "high-end"): "Here are our premium luxury properties with top-tier amenities..."
   - For additional options requests ("different property options"): "Here are different property options in new areas for you..."
   - For area diversification requests ("different areas"): "I've found properties in different locations this time..."
2. Explains what you found (or why nothing was found)
3. If properties found: Use appropriate language for the number shown and price positioning
4. If showing cheapest options: Clearly mention "These are the most affordable options in our current listings"
5. If showing luxury properties: Clearly mention "These are our premium luxury properties with the highest prices and best amenities"
6. If user requested more affordable options: Emphasize the budget-friendly nature of the results
7. If showing different areas: Mention that these are from different locations for variety
7. If no properties: Provides 2-3 helpful suggestions to adjust their search
8. Always end with: "Would you like more details about any of these properties or have other preferences?"
8. Keep it under 150 words
9. Use a warm, helpful tone but ALWAYS respond in English only. Do not mix languages or use Malay words.
10. Focus on Sabah locations but use English descriptions only

Do not include property details in your response - just the explanation.
"""
        
        response = llm.invoke(prompt)
        return response.content.strip()
        
    except Exception as e:
        print(f"Gemini response generation failed: {e}, falling back to rule-based")
        return generate_chatbot_response(user_message, properties, filters_applied, total_found)

# ===============================
# Database Query Functions
# ===============================
def build_property_query(
    db: Session,
    min_price: Optional[float] = None,
    max_price: Optional[float] = None,
    bedrooms: Optional[int] = None,
    property_type: Optional[str] = None,
    locations: List[str] = None,
    furnishing: Optional[str] = None,
    amenities: List[str] = None,
    exclude_property_ids: List[int] = None,
    prefer_different_areas: bool = False,
    is_luxury_request: bool = False,
    limit: int = 10
):
    """Build and execute property search query"""
    
    query = db.query(House)
    
    # Exclude specific property IDs
    if exclude_property_ids:
        query = query.filter(~House.id.in_(exclude_property_ids))
    
    # Price filters
    if min_price is not None:
        query = query.filter(House.price >= min_price)
    if max_price is not None:
        query = query.filter(House.price <= max_price)
    
    # Bedroom filter
    if bedrooms is not None:
        query = query.filter(House.bedroom >= bedrooms)
    
    # Property type filter
    if property_type:
        query = query.filter(House.property_type.ilike(f'%{property_type}%'))
    
    # Location filter with preference for different areas
    if locations:
        location_conditions = []
        for location in locations:
            location_conditions.append(House.address.ilike(f'%{location}%'))
        query = query.filter(or_(*location_conditions))
    elif prefer_different_areas:
        # If no specific locations but want different areas, prioritize diverse locations
        # This will be handled in the ordering logic below
        pass
    
    # Furnishing filter (handle both correct spelling and typo in database)
    if furnishing:
        if furnishing == 'Partially furnished':
            # Handle both correct spelling and typo in database
            query = query.filter(or_(
                House.furnishing_status == 'Partially furnished',
                House.furnishing_status == 'Partially funished'
            ))
        else:
            query = query.filter(House.furnishing_status == furnishing)
    
    # Amenities filter
    if amenities:
        for amenity in amenities:
            query = query.filter(House.amenities.op('?')(amenity))
    
    # Order by multiple criteria to ensure diverse results
    if is_luxury_request:
        # For luxury requests, order by highest price first to get most expensive properties
        query = query.order_by(
            House.price.desc(),
            House.property_type.asc(),
            House.id.asc()
        ).distinct()
    elif is_luxury_request:
        # For luxury requests, prioritize most expensive properties first
        query = query.order_by(
            House.price.desc(),  # Most expensive first
            House.property_type.asc(),
            House.id.asc()
        ).distinct()
    elif prefer_different_areas:
        # Randomize order to get different areas
        query = query.order_by(func.random(), House.price.asc())
    else:
        # For same-area requests, prioritize by price and property type
        # This ensures users get more options from the same area first
        query = query.order_by(
            House.price.asc(),
            House.property_type.asc(),
            House.id.asc()
        ).distinct()
    
    return query.limit(limit).all()

def diversify_property_results(properties: List[House], limit: int = 10, prefer_different_areas: bool = False) -> List[House]:
    """
    Diversify property results to avoid showing too many similar properties
    """
    if len(properties) <= 3:
        return properties
    
    diversified = []
    used_titles = set()
    used_addresses = set()
    
    if prefer_different_areas:
        # For different areas, be strict about diversification
        # First pass: Add unique properties (different titles and locations)
        for prop in properties:
            title_key = prop.title.lower().strip()
            address_key = prop.address.lower().strip()
            
            if title_key not in used_titles and address_key not in used_addresses:
                diversified.append(prop)
                used_titles.add(title_key)
                used_addresses.add(address_key)
                
                if len(diversified) >= limit:
                    break
    else:
        # For same area, be more lenient - only filter by title to avoid exact duplicates
        for prop in properties:
            title_key = prop.title.lower().strip()
            
            if title_key not in used_titles:
                diversified.append(prop)
                used_titles.add(title_key)
                
                if len(diversified) >= limit:
                    break
    
    # Second pass: If we need more properties, add remaining ones (different titles only)
    if len(diversified) < limit:
        for prop in properties:
            title_key = prop.title.lower().strip()
            
            if title_key not in used_titles:
                diversified.append(prop)
                used_titles.add(title_key)
                
                if len(diversified) >= limit:
                    break
    
    # Third pass: If still need more, add remaining properties by ID
    if len(diversified) < limit:
        used_ids = {prop.id for prop in diversified}
        for prop in properties:
            if prop.id not in used_ids:
                diversified.append(prop)
                if len(diversified) >= limit:
                    break
    
    return diversified

def generate_chatbot_response(
    user_message: str,
    properties: List[House],
    filters_applied: Dict[str, Any],
    total_found: int,
    db: Session = None
) -> str:
    """Generate AI response explaining property selection reasoning"""
    
    if total_found == 0:
        # Provide helpful suggestions when no properties are found
        suggestions_text = ""
        if filters_applied.get('max_price'):
            suggestions_text += f"• Try increasing your budget above RM {filters_applied['max_price']:,.0f}\n"
        if filters_applied.get('bedrooms'):
            suggestions_text += f"• Consider {filters_applied['bedrooms']-1} bedroom alternatives\n"
        if filters_applied.get('property_type'):
            suggestions_text += f"• Look at other property types besides {filters_applied['property_type']}\n"
        if filters_applied.get('locations'):
            suggestions_text += f"• Search in nearby areas\n"
        
        return f"""I couldn't find any properties matching "{user_message}". 

{suggestions_text if suggestions_text else ''}Try adjusting your criteria or let me know what's most important to you!"""

    # Analyze price positioning if db session is available
    price_analysis = {"is_cheapest": False}
    if db and properties:
        price_analysis = analyze_price_positioning(properties, filters_applied, db)
    
    # Determine actual number of properties shown and use appropriate language
    num_shown = len(properties)
    
    # For properties found, explain the selection reasoning
    if total_found == 1:
        if price_analysis["is_cheapest"]:
            return f"""Perfect match! I found exactly what you're looking for, and this is the most affordable option available:"""
        else:
            return f"""Perfect match! I found exactly what you're looking for:"""
    
    elif num_shown == 1:
        if price_analysis["is_cheapest"]:
            return f"""Here's the cheapest available property from {total_found} options based on your preferences:"""
        else:
            return f"""Here's your top pick from {total_found} available properties:"""
    
    elif num_shown == 2:
        if price_analysis["is_cheapest"]:
            return f"""Here are the 2 cheapest available properties from {total_found} options based on your preferences:"""
        else:
            return f"""Here are your top 2 picks from {total_found} available properties:"""
    
    elif num_shown == 3 and total_found == 3:
        if price_analysis["is_cheapest"]:
            return f"""Here are all 3 options I found, listed from most to least affordable:"""
        else:
            return f"""Here are all 3 great options I found for you:"""
    
    elif num_shown == 3 and total_found > 3:
        if price_analysis["is_cheapest"]:
            return f"""Here are the 3 cheapest available properties from {total_found} options based on your preferences:"""
        else:
            # Generate explanation for why these top 3 were selected
            explanation_parts = []
            
            # Analyze the top 3 properties to generate reasoning
            top_3 = properties[:3]
            
            # Price reasoning
            if filters_applied.get('max_price'):
                budget = filters_applied['max_price']
                avg_price = sum(float(p.price) for p in top_3) / len(top_3)
                if avg_price <= budget * 0.8:
                    explanation_parts.append("excellent value within your budget")
                else:
                    explanation_parts.append("best options within your price range")
            
            # Type/size reasoning
            if filters_applied.get('bedrooms'):
                explanation_parts.append(f"matching your {filters_applied['bedrooms']}-bedroom requirement")
            
            if filters_applied.get('property_type'):
                explanation_parts.append(f"all {filters_applied['property_type'].lower()}s as requested")
            
            # Location reasoning
            if filters_applied.get('locations'):
                explanation_parts.append("in your preferred areas")
            
            # Furnishing reasoning
            if filters_applied.get('furnishing'):
                explanation_parts.append(f"with {filters_applied['furnishing'].lower()} status")
            
            # Default reasoning if no specific filters
            if not explanation_parts:
                explanation_parts = ["offering the best combination of price, location, and amenities"]
            
            # Combine explanations
            if len(explanation_parts) == 1:
                reasoning = explanation_parts[0]
            elif len(explanation_parts) == 2:
                reasoning = f"{explanation_parts[0]} and {explanation_parts[1]}"
            else:
                reasoning = f"{', '.join(explanation_parts[:-1])}, and {explanation_parts[-1]}"
            
            return f"""I've carefully selected these top 3 properties for you based on {reasoning}. These represent the best matches from {total_found} available options:"""
    
    else:
        # Default fallback for any other cases
        if price_analysis["is_cheapest"]:
            return f"""Here are the {num_shown} cheapest available properties from {total_found} options based on your preferences:"""
        else:
            return f"""Here are your top {num_shown} picks from {total_found} available properties:"""

def detect_incomplete_query(user_message: str) -> bool:
    """
    Detect if the user's query is incomplete and needs clarification
    """
    message_lower = user_message.lower()
    
    # Check if the message has sufficient information to search for properties
    has_budget = re.search(r'\d+(?:\.\d+)?k?', message_lower) or any(amount in message_lower for amount in ['rm', 'ringgit'])
    has_family_info = any(word in message_lower for word in ['people', 'person', 'family', 'couple', 'single', 'alone'])
    has_property_type = any(word in message_lower for word in ['house', 'apartment', 'condo', 'condominium', 'property', 'unit', 'rental', 'luxury'])
    has_location = any(location in message_lower for location in ['kota kinabalu', 'kk', 'penampang', 'tuaran', 'sandakan', 'tawau', 'sabah', 'likas', 'lintas', 'kolombong', 'inanam', 'menggatal'])
    has_bedroom_info = any(word in message_lower for word in ['bedroom', 'bed', 'br', 'room'])
    
    # If the message has property type + location, it's complete enough to search
    # OR if it has budget + (family info OR property type), it's also complete
    has_minimum_info = (has_property_type and has_location) or (has_budget and (has_family_info or has_property_type))
    
    # Only flag as incomplete if it lacks minimum information
    if has_minimum_info:
        return False
    
    # Check for incomplete query patterns that need clarification
    incomplete_patterns = [
        'near my workplace', 'near my office', 'near my job', 'near my company',
        'near where i work', 'near my work place', 'close to work',
        'i want a house', 'i want an apartment', 'i want a condo',
        'find me a', 'show me a', 'i need a', 'looking for a',
        'near the city', 'near town', 'near downtown', 'near center',
        'in a good area', 'in a nice area', 'in a safe area',
        'with parking', 'with pool', 'with gym', 'with security',
        'for my family', 'for my kids', 'for my parents'
    ]
    
    # Check if it lacks specific information that would make it complete
    lacks_specific_info = (
        ('workplace' in message_lower or 'office' in message_lower or 'work' in message_lower) and 
        not any(location in message_lower for location in ['kota kinabalu', 'kk', 'penampang', 'tuaran', 'sandakan', 'tawau', 'likas', 'lintas', 'kolombong', 'inanam', 'menggatal', 'address', 'location']) and
        not re.search(r'menara|wisma|building|tower', message_lower)  # Don't flag as incomplete if workplace building is mentioned
    ) or (
        # Improved budget detection - only flag as incomplete if budget is mentioned but no amount is provided
        ('budget' in message_lower and 
         not re.search(r'\d+(?:\.\d+)?k?', message_lower) and  # Check for any number pattern (including 1900, 2k, etc.)
         not any(amount in message_lower for amount in ['rm', 'ringgit']) and
         not any(comparison in message_lower for comparison in ['more than', 'less than', 'over', 'under', 'above', 'below', 'around', 'maximum', 'max']))
    ) or (
        ('house' in message_lower or 'apartment' in message_lower) and 
        not any(spec in message_lower for spec in ['bedroom', 'bathroom', 'bed', 'bath', 'room'])
    )
    
    # Check if message contains incomplete patterns AND lacks specific details
    has_incomplete_pattern = any(pattern in message_lower for pattern in incomplete_patterns) and lacks_specific_info
    
    return has_incomplete_pattern or lacks_specific_info

def generate_clarification_response(user_message: str) -> str:
    """
    Generate clarifying questions based on incomplete user query
    """
    message_lower = user_message.lower()
    
    # Determine what information is missing and ask relevant questions
    missing_info = []
    clarifying_questions = []
    
    # Check for missing location/workplace info
    if any(word in message_lower for word in ['workplace', 'office', 'work', 'job', 'company']) and \
       not any(location in message_lower for location in ['kota kinabalu', 'kk', 'penampang', 'tuaran', 'sandakan', 'tawau', 'address', 'location']):
        missing_info.append("workplace location")
        clarifying_questions.append("📍 **Where is your workplace located?** (e.g., Kota Kinabalu, Penampang, Tuaran)")
    
    # Check for missing budget info - improved detection
    if ('budget' in message_lower and 
        not any(amount in message_lower for amount in ['rm', 'ringgit', '2000', '3000', '4000', '5000', '1500', '2500', '3500', '4500', '1000', '6000', '7000', '8000', '9000', '10000']) and
        not any(comparison in message_lower for comparison in ['more than', 'less than', 'over', 'under', 'above', 'below'])):
        missing_info.append("budget")
        clarifying_questions.append("💰 **What's your monthly budget for rent?** (e.g., RM 1500, RM 2000, RM 3000)")
    
    # Check for missing property details
    if any(prop_type in message_lower for prop_type in ['house', 'apartment', 'condo', 'property']) and \
       not any(spec in message_lower for spec in ['bedroom', 'bathroom', 'bed', 'bath', 'room', 'studio']):
        missing_info.append("property specifications")
        clarifying_questions.append("🏠 **How many bedrooms do you need?** (e.g., 1 bedroom, 2 bedrooms, 3 bedrooms)")
    
    # Check for missing family size info
    if any(word in message_lower for word in ['family', 'kids', 'children', 'parents']) and \
       not any(size in message_lower for size in ['big', 'small', 'large', 'couple', 'single']):
        missing_info.append("family size")
        clarifying_questions.append("👨‍👩‍👧‍👦 **How many people will be living in the property?**")
    
    # Check for missing amenities
    if any(word in message_lower for word in ['amenities', 'facilities', 'features']) and \
       not any(amenity in message_lower for amenity in ['parking', 'pool', 'gym', 'security', 'lift']):
        missing_info.append("preferred amenities")
        clarifying_questions.append("🏊‍♂️ **What amenities are important to you?** (e.g., Parking, Swimming Pool, Gym, 24hr Security)")
    
    # Generate response based on missing information
    if missing_info:
        response = f"""I'd be happy to help you find the perfect property! 🤗

To give you the most accurate recommendations, I need a few more details:

{chr(10).join(clarifying_questions)}

Once you provide these details, I can search for properties that match your specific needs perfectly! 😊"""
        
        return response
    
    # Fallback response if we can't determine what's missing
    return """I'd be happy to help you find the perfect property! 🤗

To give you the most accurate recommendations, could you please provide more details such as:

📍 **Preferred location** (e.g., Kota Kinabalu, Penampang, Tuaran)
💰 **Your budget** (e.g., RM 1500, RM 2000, RM 3000)
🏠 **Number of bedrooms** (e.g., 1, 2, 3 bedrooms)
👨‍👩‍👧‍👦 **Family size** (how many people will live there)
🏊‍♂️ **Preferred amenities** (e.g., Parking, Pool, Gym, Security)

Once you provide these details, I can search for properties that match your specific needs perfectly! 😊"""

def detect_general_question(user_message: str) -> bool:
    """
    Detect if the user is asking general questions (not property search related)
    """
    message_lower = user_message.lower()
    
    # Check for general question keywords
    general_keywords = [
        'what is', 'what are', 'how does', 'how do', 'tell me about', 'explain',
        'information about', 'help me understand', 'can you explain', 'what means',
        'definition of', 'difference between', 'compare', 'pros and cons',
        'advantages', 'disadvantages', 'benefits', 'risks'
    ]
    
    # Check if it's a general question but not property search related
    has_general_keywords = any(keyword in message_lower for keyword in general_keywords)
    
    # Make sure it's not a property search question
    property_search_keywords = ['property', 'house', 'apartment', 'condo', 'rent', 'rental', 'budget', 'bedroom', 'bathroom']
    is_property_search = any(keyword in message_lower for keyword in property_search_keywords)
    
    return has_general_keywords and not is_property_search

def detect_rental_process_question(user_message: str) -> bool:
    """
    Detect if the user is asking about rental process/how to secure a property
    """
    message_lower = user_message.lower()
    
    # Check for rental process keywords - expanded list
    rental_process_keywords = [
        'how to secure', 'how can i secure', 'how can do i secure', 'how do i secure',
        'how to rent', 'how to book', 'how to apply', 'how to get',
        'what do i need', 'what documents', 'what steps', 'what process',
        'rental process', 'booking process', 'application process', 'securing process',
        'steps to rent', 'steps to secure', 'requirements', 'procedure',
        'deposit', 'contract', 'agreement', 'lease', 'tenant', 'landlord',
        'rental agreement', 'rental contract', 'security deposit', 'advance payment',
        'how much deposit', 'rental terms', 'rental conditions', 'rental procedure',
        'secure this', 'secure the', 'rent this', 'rent the', 'book this', 'book the',
        'apply for', 'application for', 'get this', 'get the',
        'can i secure', 'can i rent', 'can i book', 'can i apply',
        'process to', 'steps to', 'requirements to', 'what is needed to',
        'how much to', 'how long to', 'how do i', 'what should i',
        'rental guide', 'renting guide', 'booking guide', 'application guide'
    ]
    
    # Check if any rental process keywords are present
    has_rental_keywords = any(keyword in message_lower for keyword in rental_process_keywords)
    
    return has_rental_keywords

def detect_luxury_request(user_message: str, extracted_params: Dict[str, Any]) -> bool:
    """
    Detect if the user is requesting luxury/high-end properties
    """
    message_lower = user_message.lower()
    
    # Check for luxury keywords
    luxury_keywords = [
        'luxury', 'high-end', 'premium', 'expensive', 'most expensive', 
        'top-tier', 'upscale', 'exclusive', 'deluxe', 'lavish', 'opulent', 
        'grand', 'sophisticated', 'elite', 'prestigious', 'posh', 'fancy', 
        'extravagant', 'high-class', 'upper-class', 'wealthy', 'rich', 
        'costly', 'pricey', 'high-priced', 'top-end', 'best', 'finest', 
        'superior', 'exceptional', 'outstanding', 'premium property',
        'luxury house', 'luxury condo', 'luxury apartment', 'expensive properties',
        'show me expensive', 'most expensive', 'top properties', 'best properties',
        'expensive house', 'high-end house', 'deluxe property', 'luxury rental',
        'expensive rental', 'premium rental'
    ]
    
    # Check if any luxury keywords are present
    has_luxury_keywords = any(keyword in message_lower for keyword in luxury_keywords)
    
    # Check if min_price is set to a high value (luxury threshold)
    has_high_min_price = extracted_params.get('min_price') and extracted_params['min_price'] >= 5000
    
    return has_luxury_keywords or has_high_min_price

def generate_rental_process_guidance(user_message: str) -> str:
    """
    Generate comprehensive rental process guidance following Malaysian standards
    """
    message_lower = user_message.lower()
    
    # Customize response based on specific question
    if 'deposit' in message_lower or 'how much' in message_lower:
        intro = "💰 **Malaysian Rental Deposit & Payment Standards**\n\nHere's what you need to know about deposits and payments according to Malaysian rental practices:"
    elif 'documents' in message_lower or 'requirements' in message_lower:
        intro = "📋 **Required Documents for Malaysian Rental**\n\nHere are the documents you'll need to secure a rental property in Malaysia:"
    elif 'process' in message_lower or 'steps' in message_lower:
        intro = "📝 **Malaysian Rental Process Steps**\n\nHere's the complete step-by-step process following Malaysian rental standards:"
    else:
        intro = "🏠 **How to Secure Your Rental Property in Malaysia (Sabah)**\n\nHere's a complete guide following Malaysian rental standards and legal requirements:"

    return f"""{intro}

**📋 Step 1: Malaysian Rental Requirements**
• **Malaysian IC** or **Valid Passport** (for foreigners)
• **Latest 3 months salary slips** (minimum RM 3,000 income recommended)
• **Bank statements** (last 3 months showing regular income)
• **Employment letter** from current employer
• **EPF statement** (for Malaysian employees)
• **Reference letter** from previous landlord (if applicable)
• **Guarantor documents** (if income is insufficient)

**💰 Step 2: Malaysian Rental Financial Standards**
• **Security Deposit**: 2-3 months' rent (standard practice)
• **Advance Payment**: 1-2 months' rent upfront
• **Utility Deposits**: 
  - Water deposit: RM 200-500
  - Electricity deposit: RM 400-800
  - Internet/TNB deposit: RM 200-400
• **Agent Commission**: 1 month's rent (if using property agent)
• **Legal fees**: RM 200-500 (for tenancy agreement)

**📝 Step 3: Malaysian Rental Application Process**
1. **Property Viewing**: Schedule with landlord/agent
2. **Application Form**: Complete rental application
3. **Document Submission**: Provide all required documents
4. **Background Verification**: Landlord verifies employment & income
5. **Approval Decision**: Usually 3-7 working days
6. **Agreement Preparation**: Draft tenancy agreement

**📄 Step 4: Malaysian Tenancy Agreement (Standard Terms)**
• **Contract Duration**: Minimum 1 year (standard practice)
• **Rent Amount**: Fixed monthly rent (no increase during contract)
• **Deposit Structure**: Clearly stated security deposit terms
• **Utilities Responsibility**: Who pays for water, electricity, internet
• **Maintenance Terms**: Landlord vs tenant responsibilities
• **Notice Period**: 2-3 months notice for termination
• **Stamp Duty**: RM 10 per RM 1,000 rental (tenant's responsibility)

**🔑 Step 5: Malaysian Move-in Process**
• **Final Property Inspection**: Document existing condition
• **Key & Access Handover**: Receive all keys, access cards
• **Utility Transfer**: Transfer utilities to tenant's name
• **Inventory Checklist**: Document all furniture & appliances
• **Condition Report**: Photos of property condition
• **Emergency Contacts**: Landlord/agent contact information

**🇲🇾 Malaysian Rental Legal Standards:**
• **Stamp Duty**: Required by Malaysian law (RM 10 per RM 1,000)
• **Tenancy Act 1966**: Governs rental agreements in Malaysia
• **Notice Period**: 2-3 months written notice for termination
• **Deposit Return**: Within 30 days after contract ends
• **Rent Increase**: Only after contract renewal (not during contract)

**💡 Malaysian Rental Best Practices:**
• **Always verify landlord ownership** before paying deposits
• **Get stamped tenancy agreement** (legal requirement)
• **Keep all receipts** for deposits and payments
• **Understand utility billing** (some landlords include, others don't)
• **Know your rights** under Malaysian Tenancy Act
• **Document everything** in case of disputes

**⚠️ Important Malaysian Considerations:**
• **Foreigners**: May need additional documentation and higher deposits
• **Student Rentals**: Often require guarantor (Malaysian citizen)
• **Short-term Rentals**: May have different terms and higher rates
• **Furnished vs Unfurnished**: Different deposit requirements

**Need Help?** Contact the property owner, agent, or consult a Malaysian property lawyer for specific legal advice."""

def generate_general_response(user_message: str) -> str:
    """
    Generate appropriate response for general questions without property recommendations
    """
    return """I'm a property assistant focused on helping you find rental properties in Sabah, Malaysia. 

For general questions about real estate, property investment, or housing market information, I'd recommend:

• **Property Investment**: Consult with a licensed real estate agent or financial advisor
• **Market Information**: Check with local property websites or real estate agencies
• **Legal Questions**: Contact a property lawyer for legal advice
• **General Housing Info**: Visit government housing websites or local authorities

If you're looking to **find rental properties**, I can help you search based on your preferences like:
• Budget and price range
• Number of bedrooms and bathrooms  
• Property type (apartment, house, condo)
• Preferred locations in Sabah
• Amenities and features

Would you like to search for rental properties instead?"""

def generate_suggestions(user_message: str, properties: List[House]) -> List[str]:
    """Generate helpful suggestions for the user"""
    suggestions = []
    
    if not properties:
        suggestions = [
            "My budget is more than RM 2,500",
            "My budget is less than RM 2,500",
            "I have a big family",
            "I have a small family",
            "Rental near city centre"
        ]
    else:
        # Return empty suggestions after AI gives results
        suggestions = []
    
    return suggestions

# ===============================
# Main Chatbot Endpoint
# ===============================
@router.post("/search", response_model=ChatbotResponse)
def chatbot_property_search(
    query: ChatbotQuery,
    db: Session = Depends(get_db)
):
    """
    AI-powered property search endpoint that understands natural language queries
    """
    try:
        user_message = query.user_message.strip()
        
        # Check if this is a rental process question first
        if detect_rental_process_question(user_message):
            rental_guidance = generate_rental_process_guidance(user_message)
            
            # Generate conversation ID if not provided
            conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
            
            return ChatbotResponse(
                message=rental_guidance,
                properties=[],  # No properties for rental guidance
                conversation_id=conversation_id,
                filters_applied={},
                total_found=0,
                suggestions=[]  # No suggestions for rental guidance
            )
        
        # Check if user is requesting properties outside Sabah
        # But skip this check if it's a "More Options" request
        is_more_options_request = any(phrase in user_message.lower() for phrase in [
            'more sabah property options', 'more property options', 'show me more', 
            'different areas', 'exclude property ids', 'more options'
        ])
        
        if not is_more_options_request:
            non_sabah_location = detect_non_sabah_location(user_message)
            if non_sabah_location:
                non_sabah_response = generate_non_sabah_location_response(user_message, non_sabah_location)
                
                # Generate conversation ID if not provided
                conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
                
                return ChatbotResponse(
                    message=non_sabah_response,
                    properties=[],  # No properties for non-Sabah requests
                    conversation_id=conversation_id,
                    filters_applied={},
                    total_found=0,
                    suggestions=[]  # No suggestions for non-Sabah requests
                )
        
        # Check if user message is gibberish
        if detect_gibberish(user_message):
            gibberish_response = generate_gibberish_response(user_message)
            
            # Generate conversation ID if not provided
            conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
            
            return ChatbotResponse(
                message=gibberish_response,
                properties=[],  # No properties for gibberish
                conversation_id=conversation_id,
                filters_applied={},
                total_found=0,
                suggestions=[]  # No suggestions for gibberish
            )
        
        # Check if this is an incomplete query that needs clarification
        if detect_incomplete_query(user_message):
            clarification_response = generate_clarification_response(user_message)
            
            # Generate conversation ID if not provided
            conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
            
            return ChatbotResponse(
                message=clarification_response,
                properties=[],  # No properties for clarification requests
                conversation_id=conversation_id,
                filters_applied={},
                total_found=0,
                suggestions=[]  # No suggestions for clarification requests
            )
        
        # Check if this is a general question (not property search related)
        if detect_general_question(user_message):
            general_response = generate_general_response(user_message)
            
            # Generate conversation ID if not provided
            conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
            
            return ChatbotResponse(
                message=general_response,
                properties=[],  # No properties for general questions
                conversation_id=conversation_id,
                filters_applied={},
                total_found=0,
                suggestions=[
                    "Show me properties under RM 2000",
                    "Find 3 bedroom houses in Kota Kinabalu", 
                    "I need affordable apartments with parking",
                    "What documents do I need to rent a house?"
                ]
            )
        
        # Extract search parameters using Gemini AI (with rule-based fallback)
        extracted_params = extract_parameters_with_gemini(user_message)
        
        min_price = extracted_params["min_price"]
        max_price = extracted_params["max_price"]
        bedrooms = extracted_params["bedrooms"]
        property_type = extracted_params["property_type"]
        locations = extracted_params["locations"]
        furnishing = extracted_params["furnishing"]
        amenities = extracted_params["amenities"]
        
        # Build filters applied dictionary
        filters_applied = {
            "min_price": min_price,
            "max_price": max_price,
            "bedrooms": bedrooms,
            "property_type": property_type,
            "locations": locations,
            "furnishing": furnishing,
            "amenities": amenities
        }
        
        # Remove None values for cleaner response
        filters_applied = {k: v for k, v in filters_applied.items() if v is not None and v != []}
        
        # Detect if this is a luxury request
        is_luxury_request = detect_luxury_request(user_message, extracted_params)
        
        # Search properties
        raw_properties = build_property_query(
            db=db,
            min_price=min_price,
            max_price=max_price,
            bedrooms=bedrooms,
            property_type=property_type,
            locations=locations,
            furnishing=furnishing,
            amenities=amenities,
            exclude_property_ids=extracted_params.get("exclude_property_ids", []),
            prefer_different_areas=extracted_params.get("prefer_different_areas", False),
            is_luxury_request=is_luxury_request,
            limit=20  # Get more results to diversify from
        )
        
        # If no results found with strict filters, try a more relaxed search
        if not raw_properties:
            # Try without location restrictions but keep other important filters
            raw_properties = build_property_query(
                db=db,
                min_price=min_price,
                max_price=max_price,
                bedrooms=bedrooms,
                property_type=property_type,
                locations=None,  # Remove location restriction
                furnishing=None,  # Remove furnishing restriction
                amenities=None,   # Remove amenities restriction
                exclude_property_ids=extracted_params.get("exclude_property_ids", []),
                prefer_different_areas=True,  # Prefer different areas
                is_luxury_request=is_luxury_request,
                limit=20
            )
            
        # If still no results, get any available properties (last resort)
        if not raw_properties:
            raw_properties = build_property_query(
                db=db,
                exclude_property_ids=extracted_params.get("exclude_property_ids", []),
                prefer_different_areas=True,
                is_luxury_request=is_luxury_request,
                limit=20
            )
        
        # Diversify results to avoid duplicates and similar properties
        properties = diversify_property_results(raw_properties, limit=10, prefer_different_areas=extracted_params.get("prefer_different_areas", False))
        
        total_found = len(properties)
        
        # Generate natural language response using Gemini AI
        response_message = generate_response_with_gemini(
            user_message=user_message,
            properties=properties,
            filters_applied=filters_applied,
            total_found=total_found,
            db=db,
            is_luxury_request=is_luxury_request
        )
        
        # Generate suggestions
        suggestions = generate_suggestions(user_message, properties)
        
        # Generate conversation ID if not provided
        conversation_id = query.conversation_id or f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
        
        return ChatbotResponse(
            message=response_message,
            properties=[HouseResponse.from_orm(prop) for prop in properties],
            conversation_id=conversation_id,
            filters_applied=filters_applied,
            total_found=total_found,
            suggestions=suggestions
        )
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Chatbot search error: {str(e)}")


@router.get("/rental-guide")
def get_rental_guide():
    """Get comprehensive rental process guidance"""
    return {
        "message": generate_rental_process_guidance(""),
        "title": "Rental Process Guide",
        "sections": [
            "Initial Requirements",
            "Financial Preparation", 
            "Application Process",
            "Rental Agreement",
            "Move-in Process",
            "Pro Tips"
        ]
    }

@router.get("/suggestions")
def get_search_suggestions():
    """Get sample search queries for users"""
    return {
        "suggestions": [
            "My budget is more than RM 2,500",
            "My budget is less than RM 2,500", 
            "I have a big family",
            "I have a small family",
            "Rental near city centre"
        ]
    }

@router.post("/refresh")
def refresh_chat_session():
    """Refresh chat session and get new conversation ID with fresh suggestions"""
    try:
        # Generate new conversation ID
        new_conversation_id = f"conv_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
        
        # Get fresh suggestions
        fresh_suggestions = [
            "My budget is more than RM 2,500",
            "My budget is less than RM 2,500",
            "I have a big family", 
            "I have a small family",
            "Rental near city centre"
        ]
        
        welcome_message = "Hi! I'm your property assistant for Sabah. I specialize in helping with:\n\n🏠 Rental properties\n🚛 Moving services & lorry rentals\n💰 Budget planning\n📍 Location recommendations\n\nClick on the suggested questions below to get started, or ask me about properties and moving services!\n\n💡 If you prefer traditional search, minimize this chat and use the filters above."
        
        return {
            "message": "Chat refreshed successfully!",
            "conversation_id": new_conversation_id,
            "welcome_message": welcome_message,
            "suggestions": fresh_suggestions,
            "timestamp": datetime.now().isoformat()
        }
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Chat refresh error: {str(e)}")