Upload 82 files

DB_store_backup/agentic_context_convoid_management.py

@@ -0,0 +1,1157 @@
+# from fastapi import FastAPI, HTTPException, Query
+# from fastapi.middleware.cors import CORSMiddleware
+# from pydantic import BaseModel
+# from typing import Optional, List, Dict, Any
+# import psycopg2
+# from psycopg2 import sql
+# import json
+# from datetime import datetime
+# from fastapi import APIRouter
+# from fastapi.responses import JSONResponse
+
+# # PostgreSQL credentials
+# PGHOST = 'ep-steep-dream-adqtvjel-pooler.c-2.us-east-1.aws.neon.tech'
+# PGDATABASE = 'neondb'
+# PGUSER = 'neondb_owner'
+# PGPASSWORD = 'npg_Qq0B1uWRXavx'
+# PGSSLMODE = 'require'
+
+
+# # Add CORS middleware
+
+# Db_store_router= APIRouter(prefix="/Db_store_router", tags=["Db_store_router"])
+
+# # Pydantic models
+# class UserMetadata(BaseModel):
+#     location: Optional[str] = None
+#     language: Optional[str] = None
+
+# class UserQuery(BaseModel):
+#     query_id: str
+#     text: str
+#     user_metadata: Optional[UserMetadata] = None
+
+# class ArtifactMetadata(BaseModel):
+#     created_by: Optional[str] = None
+#     associated_query_id: Optional[str] = None
+#     associated_session_id: Optional[str] = None
+#     artifact_timestamp: Optional[str] = None
+
+# class Artifact(BaseModel):
+#     artifact_id: str
+#     file_id: str
+#     file_name: str
+#     file_type: str
+#     file_size: int
+#     file_url: str
+#     upload_timestamp: str
+#     metadata: Optional[ArtifactMetadata] = None
+
+# class Response(BaseModel):
+#     text: str
+#     status: str
+#     response_time: str
+#     duration: str
+#     artifacts: List[Artifact]
+
+# class conversation_data(BaseModel):
+#     user_query: UserQuery
+#     response: Response
+#     metadata: Optional[Dict[str, Any]] = None
+
+# class ConversationInput(BaseModel):
+#     convo_id: str
+#     data: conversation_data
+#     is_saved: Optional[bool] = False
+
+# class UpdateSavedStatus(BaseModel):
+#     is_saved: bool
+
+# # Database connection
+# def connect_to_db():
+#     try:
+#         conn = psycopg2.connect(
+#             host=PGHOST,
+#             database=PGDATABASE,
+#             user=PGUSER,
+#             password=PGPASSWORD,
+#             sslmode=PGSSLMODE
+#         )
+#         return conn
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Database connection error: {str(e)}")
+
+# # Database initialization
+# def create_table():
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     create_table_query = """
+#         CREATE TABLE IF NOT EXISTS "conversation_data" (
+#             id SERIAL PRIMARY KEY,
+#             convo_id VARCHAR(255) NOT NULL,
+#             user_query JSONB NOT NULL,
+#             response JSONB NOT NULL,
+#             file_metadata JSONB,
+#             is_saved BOOLEAN DEFAULT FALSE,
+#             created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+#             updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+#         )
+#     """
+
+#     create_index_query = """
+#         CREATE INDEX IF NOT EXISTS idx_convo_id_is_saved 
+#         ON "conversation_data" (convo_id, is_saved);
+#     """
+
+#     try:
+#         cursor.execute(create_table_query)
+#         cursor.execute(create_index_query)
+#         conn.commit()
+#     except Exception as e:
+#         conn.rollback()
+#         raise HTTPException(status_code=500, detail=f"Table creation error: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# # Initialize table on startup
+# @Db_store_router.on_event("startup")
+# async def startup_event():
+#     create_table()
+
+# # API Endpoints
+
+# @Db_store_router.get("/")
+# async def root():
+#     return {
+#         "message": "Conversation Data API",
+#         "version": "1.0.0",
+#         "endpoints": {
+#             "POST /conversations": "Create a new conversation",
+#             "GET /conversations/{convo_id}": "Get conversation by ID",
+#             "GET /conversations": "Get conversations with filters",
+#             "PUT /conversations/{convo_id}/saved": "Update saved status",
+#             "DELETE /conversations/{convo_id}": "Delete conversation",
+#             "GET /health": "Health check"
+#         }
+#     }
+
+# @Db_store_router.get("/health")
+# async def health_check():
+#     try:
+#         conn = connect_to_db()
+#         conn.close()
+#         return {"status": "healthy", "database": "connected"}
+#     except Exception as e:
+#         raise HTTPException(status_code=503, detail=f"Database unhealthy: {str(e)}")
+
+# @Db_store_router.post("/conversations", status_code=201)
+# async def create_conversation(conversation: ConversationInput):
+#     """Create a new conversation entry"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     insert_query = sql.SQL("""
+#         INSERT INTO "conversation_data" (convo_id, user_query, response, file_metadata, is_saved)
+#         VALUES (%s, %s, %s, %s, %s)
+#         RETURNING id, created_at
+#     """)
+
+#     # Convert data to JSON
+#     user_query = json.dumps(conversation.data.user_query.dict())
+#     response = json.dumps(conversation.data.response.dict())
+#     file_metadata = json.dumps([artifact.dict() for artifact in conversation.data.response.artifacts])
+
+#     try:
+#         cursor.execute(insert_query, (
+#             conversation.convo_id,
+#             user_query,
+#             response,
+#             file_metadata,
+#             conversation.is_saved
+#         ))
+#         result = cursor.fetchone()
+#         conn.commit()
+        
+#         return {
+#             "message": "Conversation created successfully",
+#             "convo_id": conversation.convo_id,
+#             "id": result[0],
+#             "created_at": str(result[1]),
+#             "is_saved": conversation.is_saved
+#         }
+#     except Exception as e:
+#         conn.rollback()
+#         raise HTTPException(status_code=500, detail=f"Error creating conversation: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# @Db_store_router.get("/conversations/{convo_id}")
+# async def get_conversation_by_id(
+#     convo_id: str,
+#     is_saved: Optional[bool] = Query(None, description="Filter by saved status")
+# ):
+#     """Get conversation(s) by convo_id, optionally filtered by saved status"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     if is_saved is not None:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data" 
+#             WHERE convo_id = %s AND is_saved = %s
+#             ORDER BY created_at DESC
+#         """
+#         cursor.execute(select_query, (convo_id, is_saved))
+#     else:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data" 
+#             WHERE convo_id = %s
+#             ORDER BY created_at DESC
+#         """
+#         cursor.execute(select_query, (convo_id,))
+
+#     try:
+#         results = cursor.fetchall()
+        
+#         if not results:
+#             raise HTTPException(status_code=404, detail=f"No conversations found for convo_id: {convo_id}")
+        
+#         conversations = []
+#         for result in results:
+#             stored_convo_id, user_query_json, response_json, file_metadata_json, saved, created_at, updated_at = result
+            
+#             conversations.append({
+#                 "convo_id": stored_convo_id,
+#                 "user_query": user_query_json,
+#                 "response": response_json,
+#                 "file_metadata": file_metadata_json,
+#                 "is_saved": saved,
+#                 "created_at": str(created_at),
+#                 "updated_at": str(updated_at)
+#             })
+        
+#         return {
+#             "count": len(conversations),
+#             "conversations": conversations
+#         }
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error retrieving conversation: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# @Db_store_router.get("/conversations")
+# async def get_conversations(
+#     is_saved: Optional[bool] = Query(None, description="Filter by saved status"),
+#     limit: int = Query(100, ge=1, le=1000, description="Maximum number of results"),
+#     offset: int = Query(0, ge=0, description="Number of results to skip")
+# ):
+#     """Get all conversations with optional filters"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     if is_saved is not None:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data" 
+#             WHERE is_saved = %s
+#             ORDER BY updated_at DESC
+#             LIMIT %s OFFSET %s
+#         """
+#         cursor.execute(select_query, (is_saved, limit, offset))
+#     else:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data" 
+#             ORDER BY updated_at DESC
+#             LIMIT %s OFFSET %s
+#         """
+#         cursor.execute(select_query, (limit, offset))
+
+#     try:
+#         results = cursor.fetchall()
+        
+#         if not results:
+#             return {"count": 0, "conversations": []}
+        
+#         conversations = []
+#         for result in results:
+#             stored_convo_id, user_query_json, response_json, file_metadata_json, saved, created_at, updated_at = result
+            
+#             conversations.append({
+#                 "convo_id": stored_convo_id,
+#                 "user_query": user_query_json,
+#                 "response": response_json,
+#                 "file_metadata": file_metadata_json,
+#                 "is_saved": saved,
+#                 "created_at": str(created_at),
+#                 "updated_at": str(updated_at)
+#             })
+        
+#         return {
+#             "count": len(conversations),
+#             "limit": limit,
+#             "offset": offset,
+#             "conversations": conversations
+#         }
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error retrieving conversations: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# @Db_store_router.put("/conversations/{convo_id}/saved")
+# async def update_saved_status(convo_id: str, status: UpdateSavedStatus):
+#     """Update the is_saved status for a conversation"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     update_query = """
+#         UPDATE "conversation_data" 
+#         SET is_saved = %s, updated_at = CURRENT_TIMESTAMP
+#         WHERE convo_id = %s
+#     """
+
+#     try:
+#         cursor.execute(update_query, (status.is_saved, convo_id))
+#         conn.commit()
+        
+#         if cursor.rowcount == 0:
+#             raise HTTPException(status_code=404, detail=f"No conversation found with convo_id: {convo_id}")
+        
+#         return {
+#             "message": f"Updated {cursor.rowcount} record(s)",
+#             "convo_id": convo_id,
+#             "is_saved": status.is_saved
+#         }
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         conn.rollback()
+#         raise HTTPException(status_code=500, detail=f"Error updating saved status: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# @Db_store_router.delete("/conversations/{convo_id}")
+# async def delete_conversation(
+#     convo_id: str,
+#     delete_all: bool = Query(False, description="Delete all records with this convo_id")
+# ):
+#     """Delete conversation(s) by convo_id"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     if delete_all:
+#         delete_query = 'DELETE FROM "conversation_data" WHERE convo_id = %s'
+#         cursor.execute(delete_query, (convo_id,))
+#     else:
+#         # Delete only the most recent one
+#         delete_query = """
+#             DELETE FROM "conversation_data" 
+#             WHERE id = (
+#                 SELECT id FROM "conversation_data" 
+#                 WHERE convo_id = %s 
+#                 ORDER BY created_at DESC 
+#                 LIMIT 1
+#             )
+#         """
+#         cursor.execute(delete_query, (convo_id,))
+
+#     try:
+#         conn.commit()
+        
+#         if cursor.rowcount == 0:
+#             raise HTTPException(status_code=404, detail=f"No conversation found with convo_id: {convo_id}")
+        
+#         return {
+#             "message": f"Deleted {cursor.rowcount} record(s)",
+#             "convo_id": convo_id
+#         }
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         conn.rollback()
+#         raise HTTPException(status_code=500, detail=f"Error deleting conversation: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# @Db_store_router.get("/stats")
+# async def get_statistics():
+#     """Get statistics about stored conversations"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+
+#     stats_query = """
+#         SELECT 
+#             COUNT(*) as total_conversations,
+#             COUNT(DISTINCT convo_id) as unique_convo_ids,
+#             SUM(CASE WHEN is_saved = TRUE THEN 1 ELSE 0 END) as saved_conversations,
+#             SUM(CASE WHEN is_saved = FALSE THEN 1 ELSE 0 END) as unsaved_conversations
+#         FROM "conversation_data"
+#     """
+
+#     try:
+#         cursor.execute(stats_query)
+#         result = cursor.fetchone()
+        
+#         return {
+#             "total_conversations": result[0],
+#             "unique_convo_ids": result[1],
+#             "saved_conversations": result[2],
+#             "unsaved_conversations": result[3]
+#         }
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error getting statistics: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+# # ------------------ FETCH DATA FUNCTION ------------------
+# def get_all_convo_data_filtered(is_saved=None):
+#     conn = connect_to_db()
+#     if not conn:
+#         return None
+
+#     cursor = conn.cursor()
+
+#     if is_saved is None:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data"
+#             ORDER BY updated_at DESC
+#         """
+#         params = ()
+#     else:
+#         select_query = """
+#             SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+#             FROM "conversation_data"
+#             WHERE is_saved = %s
+#             ORDER BY updated_at DESC
+#         """
+#         params = (is_saved,)
+
+#     try:
+#         cursor.execute(select_query, params)
+#         results = cursor.fetchall()
+
+#         if not results:
+#             return []
+
+#         conversations = []
+#         for result in results:
+#             stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved_flag, created_at, updated_at = result
+
+#             try:
+#                 user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+#             except Exception:
+#                 user_query = {}
+
+#             try:
+#                 response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+#             except Exception:
+#                 response = {}
+
+#             try:
+#                 file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+#             except Exception:
+#                 file_metadata = {}
+
+#             conversations.append({
+#                 "convo_id": stored_convo_id,
+#                 "user_query": user_query,
+#                 "response": response,
+#                 "file_metadata": file_metadata,
+#                 "is_saved": is_saved_flag,
+#                 "created_at": str(created_at),
+#                 "updated_at": str(updated_at)
+#             })
+
+#         return conversations
+
+#     except Exception as e:
+#         print(f"❌ Error retrieving data: {e}")
+#         return None
+
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+
+# # ------------------ FASTAPI ENDPOINT ------------------
+# @Db_store_router.get("/conversations")
+# def fetch_conversations(is_saved: str = Query(None, description="true | false | leave empty for all")):
+#     """
+#     Endpoint: GET /conversations?is_saved=true|false
+#     Returns conversations filtered by is_saved flag or all if not provided.
+#     """
+#     try:
+#         # Convert query param to Python boolean or None
+#         if is_saved is None:
+#             is_saved_value = None
+#         elif is_saved.lower() == "true":
+#             is_saved_value = True
+#         elif is_saved.lower() == "false":
+#             is_saved_value = False
+#         else:
+#             return JSONResponse(
+#                 status_code=400,
+#                 content={"error": "Invalid value for is_saved. Use true, false, or omit it."}
+#             )
+
+#         data = get_all_convo_data_filtered(is_saved_value)
+
+#         if data is None:
+#             return JSONResponse(status_code=500, content={"error": "Database query failed."})
+
+#         if len(data) == 0:
+#             return JSONResponse(status_code=404, content={"message": "No conversations found."})
+
+#         return JSONResponse(
+#             status_code=200,
+#             content={
+#                 "count": len(data),
+#                 "filter": is_saved_value,
+#                 "conversations": data
+#             }
+#         )
+
+#     except Exception as e:
+#         print(f"❌ API Error: {e}")
+#         return JSONResponse(status_code=500, content={"error": str(e)})
+
+
+from fastapi import FastAPI, HTTPException, Query , Depends
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import Optional, List, Dict, Any
+import psycopg2
+from psycopg2 import sql
+import json
+from datetime import datetime
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from backend.database import  get_db
+from sqlalchemy.orm import Session
+from datetime import timedelta, datetime, timezone
+from sqlalchemy import text
+
+# PostgreSQL credentials
+# PGHOST = 'ep-steep-dream-adqtvjel-pooler.c-2.us-east-1.aws.neon.tech'
+# PGDATABASE = 'neondb'
+# PGUSER = 'neondb_owner'
+# PGPASSWORD = 'npg_Qq0B1uWRXavx'
+# PGSSLMODE = 'require'
+
+PGHOST ='igwapp-proddb.ingenspark.com'
+PGUSER = 'postgres'
+PGPASSWORD = '0gMaLjyKeO5PUAznprQ1'
+PGDATABASE = 'mrapp-demodb'
+ 
+
+# Add CORS middleware
+Db_store_router = APIRouter(prefix="/Db_store_router", tags=["Db_store_router"])
+
+# Pydantic models
+class UserMetadata(BaseModel):
+    location: Optional[str] = None
+    language: Optional[str] = None
+
+class UserQuery(BaseModel):
+    query_id: str
+    text: str
+    user_metadata: Optional[UserMetadata] = None
+
+class ArtifactMetadata(BaseModel):
+    created_by: Optional[str] = None
+    associated_query_id: Optional[str] = None
+    associated_session_id: Optional[str] = None
+    artifact_timestamp: Optional[str] = None
+
+class Artifact(BaseModel):
+    artifact_id: str
+    file_id: str
+    file_name: str
+    file_type: str
+    file_size: int
+    file_url: str
+    upload_timestamp: str
+    metadata: Optional[ArtifactMetadata] = None
+
+class Response(BaseModel):
+    text: str
+    status: str
+    response_time: str
+    duration: str
+    artifacts: List[Artifact]
+
+class conversation_data(BaseModel):
+    user_query: UserQuery
+    response: Response
+    metadata: Optional[Dict[str, Any]] = None
+
+class ConversationInput(BaseModel):
+    user_id: str
+    convo_id: str
+    data: conversation_data
+    is_saved: Optional[bool] = False
+
+class UpdateSavedStatus(BaseModel):
+    is_saved: bool
+
+# Database connection
+def connect_to_db():
+    try:
+        conn = psycopg2.connect(
+            host=PGHOST,
+            database=PGDATABASE,
+            user=PGUSER,
+            password=PGPASSWORD
+            # sslmode=PGSSLMODE
+        )
+        return conn
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Database connection error: {str(e)}")
+
+# Database initialization
+def create_table():
+    conn = connect_to_db()
+    cursor = conn.cursor()
+
+    create_table_query = """
+        CREATE TABLE IF NOT EXISTS "conversation_data" (
+            id SERIAL PRIMARY KEY,
+            user_id VARCHAR(255) NOT NULL,
+            convo_id VARCHAR(255) NOT NULL,
+            user_query JSONB NOT NULL,
+            response JSONB NOT NULL,
+            file_metadata JSONB,
+            is_saved BOOLEAN DEFAULT FALSE,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """
+
+    create_index_query = """
+        CREATE INDEX IF NOT EXISTS idx_user_id_convo_id_is_saved 
+        ON "conversation_data" (user_id, convo_id, is_saved);
+    """
+
+    try:
+        # cursor.execute(create_table_query)
+        # cursor.execute(create_index_query)
+        conn.commit()
+    except Exception as e:
+        conn.rollback()
+        raise HTTPException(status_code=500, detail=f"Table creation error: {str(e)}")
+    finally:
+        cursor.close()
+        conn.close()
+
+# Initialize table on startup
+# @Db_store_router.on_event("startup")
+# async def startup_event():
+#     create_table()
+
+# API Endpoints
+@Db_store_router.get("/")
+async def root():
+    return {
+        "message": "Conversation Data API",
+        "version": "1.0.0",
+        "endpoints": {
+            "POST /conversations": "Create a new conversation",
+            "GET /conversations/{convo_id}": "Get conversation by ID",
+            "GET /conversations": "Get conversations with filters",
+            "PUT /conversations/{convo_id}/saved": "Update saved status",
+            "DELETE /conversations/{convo_id}": "Delete conversation",
+            "GET /health": "Health check",
+            "GET /stats": "Get conversation statistics"
+        }
+    }
+
+@Db_store_router.get("/health")
+async def health_check():
+    try:
+        conn = connect_to_db()
+        conn.close()
+        return {"status": "healthy", "database": "connected"}
+    except Exception as e:
+        raise HTTPException(status_code=503, detail=f"Database unhealthy: {str(e)}")
+
+# @Db_store_router.post("/conversations", status_code=201)
+# async def create_conversation(conversation: ConversationInput):
+#     """Create a new conversation entry"""
+#     conn = connect_to_db()
+#     cursor = conn.cursor()
+#     print("vastav conversation api hitted")
+
+#     insert_query = sql.SQL("""
+#         INSERT INTO "conversation_data" (user_id, convo_id, user_query, response, file_metadata, is_saved)
+#         VALUES (%s, %s, %s, %s, %s, %s)
+#         RETURNING id, created_at
+#     """)
+
+#     # Convert data to JSON
+#     user_query = json.dumps(conversation.data.user_query.dict())
+#     response = json.dumps(conversation.data.response.dict())
+#     file_metadata = json.dumps([artifact.dict() for artifact in conversation.data.response.artifacts])
+#     print("file_metadata", file_metadata)
+
+#     try:
+#         cursor.execute(insert_query, (
+#             conversation.user_id,
+#             conversation.convo_id,
+#             user_query,
+#             response,
+#             file_metadata,
+#             conversation.is_saved
+#         ))
+#         result = cursor.fetchone()
+#         conn.commit()
+        
+#         return {
+#             "message": "Conversation created successfully",
+#             "user_id": conversation.user_id,
+#             "convo_id": conversation.convo_id,
+#             "id": result[0],
+#             "created_at": str(result[1]),
+#             "is_saved": conversation.is_saved
+#         }
+#     except Exception as e:
+#         conn.rollback()
+#         raise HTTPException(status_code=500, detail=f"Error creating conversation: {str(e)}")
+#     finally:
+#         cursor.close()
+#         conn.close()
+
+@Db_store_router.post("/conversations", status_code=201)
+async def create_conversation(payload: dict, db: Session = Depends(get_db)):
+    """
+    Create a new conversation entry
+    """
+    try:
+        user_id = payload.get("user_id")
+        convo_id = payload.get("convo_id")
+        data = payload.get("data", {})
+        is_saved = payload.get("is_saved", False)
+
+        # Convert data fields to JSON
+        user_query = json.dumps(data.get("user_query", {}))
+        response = json.dumps(data.get("response", {}))
+
+        # Extract file_metadata (if present inside response.artifacts)
+        artifacts = data.get("response", {}).get("artifacts", [])
+        file_metadata = json.dumps(artifacts) if artifacts else json.dumps([])
+
+        created_at = datetime.now(timezone.utc)
+        updated_at = created_at
+
+        print("vastav conversation api hitted")
+
+        insert_query = text("""
+            INSERT INTO conversation_data (
+                user_id, convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            )
+            VALUES (
+                :user_id, :convo_id, :user_query, :response, :file_metadata, :is_saved, :created_at, :updated_at
+            )
+            RETURNING id, created_at
+        """)
+        print("insert_query", insert_query)
+
+        result = db.execute(insert_query, {
+            "user_id": user_id,
+            "convo_id": convo_id,
+            "user_query": user_query,
+            "response": response,
+            "file_metadata": file_metadata,
+            "is_saved": is_saved,
+            "created_at": created_at,
+            "updated_at": updated_at
+        })
+
+        db.commit()
+        inserted = result.fetchone()
+
+        return {
+            "message": "Conversation created successfully",
+            "id": str(inserted[0]),
+            "convo_id": convo_id,
+            "created_at": str(inserted[1]),
+            "is_saved": is_saved
+        }
+
+    except Exception as e:
+        db.rollback()
+        raise HTTPException(status_code=500, detail=f"Error creating conversation: {str(e)}")
+
+
+@Db_store_router.get("/conversations/{convo_id}")
+async def get_conversation_by_id(
+    convo_id: str,
+    user_id: str = Query(..., description="User ID"),
+    is_saved: Optional[bool] = Query(None, description="Filter by saved status")
+):
+    """Get conversation(s) by convo_id and user_id, optionally filtered by saved status"""
+    conn = connect_to_db()
+    cursor = conn.cursor()
+
+    if is_saved is not None:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data" 
+            WHERE convo_id = %s AND user_id = %s AND is_saved = %s
+            ORDER BY created_at DESC
+        """
+        cursor.execute(select_query, (convo_id, user_id, is_saved))
+    else:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data" 
+            WHERE convo_id = %s AND user_id = %s
+            ORDER BY created_at DESC
+        """
+        cursor.execute(select_query, (convo_id, user_id))
+
+    try:
+        results = cursor.fetchall()
+        
+        if not results:
+            raise HTTPException(status_code=404, detail=f"No conversations found for convo_id: {convo_id} and user_id: {user_id}")
+        
+        conversations = []
+        for result in results:
+            stored_convo_id, stored_user_id, user_query_json, response_json, file_metadata_json, saved, created_at, updated_at = result
+            
+            conversations.append({
+                "convo_id": stored_convo_id,
+                "user_id": stored_user_id,
+                "user_query": user_query_json,
+                "response": response_json,
+                "file_metadata": file_metadata_json,
+                "is_saved": saved,
+                "created_at": str(created_at),
+                "updated_at": str(updated_at)
+            })
+        
+        return {
+            "count": len(conversations),
+            "conversations": conversations
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error retrieving conversation: {str(e)}")
+    finally:
+        cursor.close()
+        conn.close()
+
+@Db_store_router.get("/conversations")
+async def get_conversations(
+    user_id: str = Query(..., description="User ID"),
+    is_saved: Optional[bool] = Query(None, description="Filter by saved status"),
+    limit: int = Query(100, ge=1, le=1000, description="Maximum number of results"),
+    offset: int = Query(0, ge=0, description="Number of results to skip")
+):
+    """Get all conversations for a user with optional filters"""
+    conn = connect_to_db()
+    cursor = conn.cursor()
+
+    if is_saved is not None:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data" 
+            WHERE user_id = %s AND is_saved = %s
+            ORDER BY updated_at DESC
+            LIMIT %s OFFSET %s
+        """
+        cursor.execute(select_query, (user_id, is_saved, limit, offset))
+    else:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data" 
+            WHERE user_id = %s
+            ORDER BY updated_at DESC
+            LIMIT %s OFFSET %s
+        """
+        cursor.execute(select_query, (user_id, limit, offset))
+
+    try:
+        results = cursor.fetchall()
+        
+        if not results:
+            return {"count": 0, "conversations": []}
+        
+        conversations = []
+        for result in results:
+            stored_convo_id, stored_user_id, user_query_json, response_json, file_metadata_json, saved, created_at, updated_at = result
+            
+            conversations.append({
+                "convo_id": stored_convo_id,
+                "user_id": stored_user_id,
+                "user_query": user_query_json,
+                "response": response_json,
+                "file_metadata": file_metadata_json,
+                "is_saved": saved,
+                "created_at": str(created_at),
+                "updated_at": str(updated_at)
+            })
+        
+        return {
+            "count": len(conversations),
+            "limit": limit,
+            "offset": offset,
+            "conversations": conversations
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error retrieving conversations: {str(e)}")
+    finally:
+        cursor.close()
+        conn.close()
+
+@Db_store_router.get("/conversations", response_class=JSONResponse)
+async def fetch_conversations(
+    user_id: str = Query(..., description="User ID"),
+    is_saved: Optional[str] = Query(None, description="true | false | leave empty for all")
+):
+    """
+    Endpoint: GET /conversations?user_id={user_id}&is_saved=true|false
+    Returns conversations filtered by user_id and is_saved flag or all if not provided.
+    """
+    try:
+        # Convert query param to Python boolean or None
+        if is_saved is None:
+            is_saved_value = None
+        elif is_saved.lower() == "true":
+            is_saved_value = True
+        elif is_saved.lower() == "false":
+            is_saved_value = False
+        else:
+            return JSONResponse(
+                status_code=400,
+                content={"error": "Invalid value for is_saved. Use true, false, or omit it."}
+            )
+
+        data = get_all_convo_data_filtered(user_id, is_saved_value)
+
+        if data is None:
+            return JSONResponse(status_code=500, content={"error": "Database query failed."})
+
+        if len(data) == 0:
+            return JSONResponse(status_code=404, content={"message": f"No conversations found for user_id: {user_id}."})
+
+        return JSONResponse(
+            status_code=200,
+            content={
+                "count": len(data),
+                "filter": {"user_id": user_id, "is_saved": is_saved_value},
+                "conversations": data
+            }
+        )
+    except Exception as e:
+        print(f"❌ API Error: {e}")
+        return JSONResponse(status_code=500, content={"error": str(e)})
+    
+@Db_store_router.delete("/conversations/{convo_id}")
+async def delete_conversation(
+    convo_id: str,
+    user_id: str = Query(..., description="User ID"),
+    delete_all: bool = Query(False, description="Delete all records with this convo_id for the user")
+):
+    """Delete conversation(s) by convo_id and user_id"""
+    conn = connect_to_db()
+    cursor = conn.cursor()
+
+    if delete_all:
+        delete_query = 'DELETE FROM "conversation_data" WHERE convo_id = %s AND user_id = %s'
+        cursor.execute(delete_query, (convo_id, user_id))
+    else:
+        # Delete only the most recent one
+        delete_query = """
+            DELETE FROM "conversation_data" 
+            WHERE id = (
+                SELECT id FROM "conversation_data" 
+                WHERE convo_id = %s AND user_id = %s
+                ORDER BY created_at DESC 
+                LIMIT 1
+            )
+        """
+        cursor.execute(delete_query, (convo_id, user_id))
+
+    try:
+        conn.commit()
+        
+        if cursor.rowcount == 0:
+            raise HTTPException(status_code=404, detail=f"No conversation found with convo_id: {convo_id} and user_id: {user_id}")
+        
+        return {
+            "message": f"Deleted {cursor.rowcount} record(s)",
+            "user_id": user_id,
+            "convo_id": convo_id
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        conn.rollback()
+        raise HTTPException(status_code=500, detail=f"Error deleting conversation: {str(e)}")
+    finally:
+        cursor.close()
+        conn.close()
+
+@Db_store_router.get("/stats")
+async def get_statistics(user_id: str = Query(..., description="User ID")):
+    """Get statistics about stored conversations for a user"""
+    conn = connect_to_db()
+    cursor = conn.cursor()
+
+    stats_query = """
+        SELECT 
+            COUNT(*) as total_conversations,
+            COUNT(DISTINCT convo_id) as unique_convo_ids,
+            SUM(CASE WHEN is_saved = TRUE THEN 1 ELSE 0 END) as saved_conversations,
+            SUM(CASE WHEN is_saved = FALSE THEN 1 ELSE 0 END) as unsaved_conversations
+        FROM "conversation_data"
+        WHERE user_id = %s
+    """
+
+    try:
+        cursor.execute(stats_query, (user_id,))
+        result = cursor.fetchone()
+        
+        return {
+            "user_id": user_id,
+            "total_conversations": result[0],
+            "unique_convo_ids": result[1],
+            "saved_conversations": result[2],
+            "unsaved_conversations": result[3]
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error getting statistics: {str(e)}")
+    finally:
+        cursor.close()
+        conn.close()
+
+# ------------------ FETCH DATA FUNCTION ------------------
+def get_all_convo_data_filtered(user_id: str, is_saved: Optional[bool] = None):
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    if is_saved is None:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data"
+            WHERE user_id = %s
+            ORDER BY updated_at DESC
+        """
+        params = (user_id,)
+    else:
+        select_query = """
+            SELECT convo_id, user_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "conversation_data"
+            WHERE user_id = %s AND is_saved = %s
+            ORDER BY updated_at DESC
+        """
+        params = (user_id, is_saved)
+
+    try:
+        cursor.execute(select_query, params)
+        results = cursor.fetchall()
+
+        if not results:
+            return []
+
+        conversations = []
+        for result in results:
+            stored_convo_id, stored_user_id, user_query_json, response_json, file_metadata_json, is_saved_flag, created_at, updated_at = result
+
+            try:
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+            except Exception:
+                user_query = {}
+
+            try:
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+            except Exception:
+                response = {}
+
+            try:
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+            except Exception:
+                file_metadata = {}
+
+            conversations.append({
+                "convo_id": stored_convo_id,
+                "user_id": stored_user_id,
+                "user_query": user_query,
+                "response": response,
+                "file_metadata": file_metadata,
+                "is_saved": is_saved_flag,
+                "created_at": str(created_at),
+                "updated_at": str(updated_at)
+            })
+
+        return conversations
+
+    except Exception as e:
+        print(f"❌ Error retrieving data: {e}")
+        return None
+
+    finally:
+        cursor.close()
+        conn.close()
+
+# ------------------ FASTAPI ENDPOINT ------------------
+@Db_store_router.get("/conversations", response_class=JSONResponse)
+async def fetch_conversations(
+    user_id: str = Query(..., description="User ID"),
+    is_saved: Optional[str] = Query(None, description="true | false | leave empty for all")
+):
+    """
+    Endpoint: GET /conversations?user_id={user_id}&is_saved=true|false
+    Returns conversations filtered by user_id and is_saved flag or all if not provided.
+    """
+    try:
+        # Convert query param to Python boolean or None
+        if is_saved is None:
+            is_saved_value = None
+        elif is_saved.lower() == "true":
+            is_saved_value = True
+        elif is_saved.lower() == "false":
+            is_saved_value = False
+        else:
+            return JSONResponse(
+                status_code=400,
+                content={"error": "Invalid value for is_saved. Use true, false, or omit it."}
+            )
+
+        data = get_all_convo_data_filtered(user_id, is_saved_value)
+
+        if data is None:
+            return JSONResponse(status_code=500, content={"error": "Database query failed."})
+
+        if len(data) == 0:
+            return JSONResponse(status_code=404, content={"message": f"No conversations found for user_id: {user_id}."})
+
+        return JSONResponse(
+            status_code=200,
+            content={
+                "count": len(data),
+                "filter": {"user_id": user_id, "is_saved": is_saved_value},
+                "conversations": data
+            }
+        )
+
+    except Exception as e:
+        print(f"❌ API Error: {e}")
+        return JSONResponse(status_code=500, content={"error": str(e)})

DB_store_backup/rough.py

@@ -0,0 +1,510 @@
+import psycopg2
+from psycopg2 import sql
+import json
+
+# PostgreSQL credentials
+PGHOST = 'ep-steep-dream-adqtvjel-pooler.c-2.us-east-1.aws.neon.tech'
+PGDATABASE = 'neondb'
+PGUSER = 'neondb_owner'
+PGPASSWORD = 'npg_Qq0B1uWRXavx'
+PGSSLMODE = 'require'
+
+# Function to connect to the PostgreSQL database
+def connect_to_db():
+    try:
+        conn = psycopg2.connect(
+            host=PGHOST,
+            database=PGDATABASE,
+            user=PGUSER,
+            password=PGPASSWORD,
+            sslmode=PGSSLMODE
+        )
+        return conn
+    except Exception as e:
+        print(f"Error connecting to PostgreSQL database: {e}")
+        return None
+    
+# Function to delete the table 
+def delete_table_by_name(table_name: str):
+    conn = connect_to_db()
+    if not conn:
+        print("Failed to connect to the database")
+        return
+
+    cursor = conn.cursor()
+
+    # Create the DROP TABLE query dynamically using the table_name argument
+    drop_table_query = sql.SQL("DROP TABLE IF EXISTS {}").format(sql.Identifier(table_name))
+
+    try:
+        cursor.execute(drop_table_query)
+        conn.commit()
+        print(f"Table '{table_name}' deleted successfully.")
+    except Exception as e:
+        conn.rollback()
+        print(f"Error deleting table '{table_name}': {e}")
+    finally:
+        cursor.close()
+        conn.close()
+
+
+
+# Function to create the table if it doesn't exist
+def create_table():
+    conn = connect_to_db()
+    if not conn:
+        return False
+
+    cursor = conn.cursor()
+
+    create_table_query = """
+        CREATE TABLE IF NOT EXISTS "stored_convoId_data" (
+            id SERIAL PRIMARY KEY,
+            convo_id VARCHAR(255) NOT NULL,
+            user_query JSONB NOT NULL,
+            response JSONB NOT NULL,
+            file_metadata JSONB,
+            is_saved BOOLEAN DEFAULT FALSE,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """
+
+    # Create index for faster queries
+    create_index_query = """
+        CREATE INDEX IF NOT EXISTS idx_convo_id_is_saved 
+        ON "stored_convoId_data" (convo_id, is_saved);
+    """
+
+    try:
+        cursor.execute(create_table_query)
+        cursor.execute(create_index_query)
+        conn.commit()
+        print("Table 'stored_convoId_data' created successfully or already exists.")
+        return True
+    except Exception as e:
+        print(f"Error creating table: {e}")
+        conn.rollback()
+        return False
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to insert the data into PostgreSQL table
+def insert_convo_data(convo_id, data, is_saved=False):
+    conn = connect_to_db()
+    if not conn:
+        return
+
+    cursor = conn.cursor()
+
+    insert_query = sql.SQL("""
+        INSERT INTO "stored_convoId_data" (convo_id, user_query, response, file_metadata, is_saved)
+        VALUES (%s, %s, %s, %s, %s)
+    """)
+
+    # Prepare data to insert (use json.dumps for JSONB columns)
+    user_query = json.dumps(data["user_query"])
+    response = json.dumps(data["response"])
+    file_metadata = json.dumps(data["response"]["artifacts"])
+
+    try:
+        cursor.execute(insert_query, (convo_id, user_query, response, file_metadata, is_saved))
+        conn.commit()
+        print(f"Data for convo_id {convo_id} inserted successfully with is_saved={is_saved}.")
+    except Exception as e:
+        print(f"Error inserting data: {e}")
+        conn.rollback()
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to update the is_saved flag
+def update_saved_status(convo_id, is_saved=True):
+    conn = connect_to_db()
+    if not conn:
+        return False
+
+    cursor = conn.cursor()
+
+    update_query = """
+        UPDATE "stored_convoId_data" 
+        SET is_saved = %s, updated_at = CURRENT_TIMESTAMP
+        WHERE convo_id = %s
+    """
+
+    try:
+        cursor.execute(update_query, (is_saved, convo_id))
+        conn.commit()
+        if cursor.rowcount > 0:
+            print(f"Updated {cursor.rowcount} record(s) for convo_id {convo_id} to is_saved={is_saved}.")
+            return True
+        else:
+            print(f"No records found for convo_id {convo_id}.")
+            return False
+    except Exception as e:
+        print(f"Error updating saved status: {e}")
+        conn.rollback()
+        return False
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to retrieve data by convo_id only
+def get_convo_data(convo_id):
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE convo_id = %s
+        ORDER BY created_at DESC
+    """
+
+    try:
+        cursor.execute(select_query, (convo_id,))
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved, created_at, updated_at = result
+                
+                # JSONB columns are already returned as dict/list by psycopg2
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print(f"No data found for convo_id: {convo_id}")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving data: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to retrieve data by convo_id and is_saved status
+def get_convo_data_by_saved_status(convo_id, is_saved=True):
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE convo_id = %s AND is_saved = %s
+        ORDER BY created_at DESC
+    """
+
+    try:
+        cursor.execute(select_query, (convo_id, is_saved))
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved_flag, created_at, updated_at = result
+                
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved_flag,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print(f"No data found for convo_id: {convo_id} with is_saved={is_saved}")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving data: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to get all saved conversations (across all convo_ids)
+def get_all_saved_conversations():
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE is_saved = TRUE
+        ORDER BY updated_at DESC
+    """
+
+    try:
+        cursor.execute(select_query)
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved, created_at, updated_at = result
+                
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print("No saved conversations found.")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving saved conversations: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Sample data
+data = {
+  "user_query": {
+    "query_id": "12345",
+    "text": "What is the weather like today?",
+    "user_metadata": {
+      "location": "New York",
+      "language": "en"
+    }
+  },
+  "response": {
+    "text": "The weather today in New York is sunny and 75°F. Would you like to see a detailed report or a forecast?",
+    "status": "success",
+    "response_time": "2025-10-13T08:10:00Z",
+    "duration": "2s",
+    "artifacts": [
+      {
+        "artifact_id": "artifact_1",
+        "file_id": "file_1",
+        "file_name": "weather_report.pdf",
+        "file_type": "pdf",
+        "file_size": 1024,
+        "file_url": "path_to_file",
+        "upload_timestamp": "2025-10-13T08:00:00Z",
+        "metadata": {
+          "created_by": "system",
+          "associated_query_id": "12345",
+          "associated_session_id": "session_001",
+          "artifact_timestamp": "2025-10-13T08:10:00Z"
+        }
+      },
+      {
+        "artifact_id": "artifact_2",
+        "file_id": "file_2",
+        "file_name": "weather_forecast_image.jpg",
+        "file_type": "jpg",
+        "file_size": 2048,
+        "file_url": "path_to_image",
+        "upload_timestamp": "2025-10-13T08:05:00Z",
+        "metadata": {
+          "created_by": "system",
+          "associated_query_id": "12345",
+          "associated_session_id": "session_001",
+          "artifact_timestamp": "2025-10-13T08:10:00Z"
+        }
+      }
+    ]
+  },
+  "metadata": {
+    "query_response_metadata": {
+      "response_time": "2025-10-13T08:10:00Z",
+      "response_status": "success",
+      "response_duration": "2s"
+    },
+    "file_metadata": [
+      {
+        "file_id": "file_1",
+        "file_name": "weather_report.pdf",
+        "file_size": 1024,
+        "upload_timestamp": "2025-10-13T08:00:00Z"
+      },
+      {
+        "file_id": "file_2",
+        "file_name": "weather_forecast_image.jpg",
+        "file_size": 2048,
+        "upload_timestamp": "2025-10-13T08:05:00Z"
+      }
+    ]
+  }
+}
+
+def get_all_convo_data_filtered(is_saved=None):
+    """
+    Retrieve all conversations across all convo_ids.
+    - If is_saved=True  → returns only saved conversations
+    - If is_saved=False → returns only unsaved conversations
+    - If is_saved=None  → returns all conversations
+    """
+    conn = connect_to_db()
+    if not conn:
+        return None
+    cursor = conn.cursor()
+
+    # Build the query dynamically
+    if is_saved is None:
+        select_query = """
+            SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "stored_convoId_data"
+            ORDER BY updated_at DESC
+        """
+        params = ()
+    else:
+        select_query = """
+            SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+            FROM "stored_convoId_data"
+            WHERE is_saved = %s
+            ORDER BY updated_at DESC
+        """
+        params = (is_saved,)
+
+    try:
+        cursor.execute(select_query, params)
+        results = cursor.fetchall()
+        if not results:
+            print(f"No conversations found with is_saved={is_saved}")
+            return None
+
+        conversations = []
+        for result in results:
+            stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved_flag, created_at, updated_at = result
+
+            user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+            response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+            file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+
+            conversations.append({
+                "convo_id": stored_convo_id,
+                "user_query": user_query,
+                "response": response,
+                "file_metadata": file_metadata,
+                "is_saved": is_saved_flag,
+                "created_at": str(created_at),
+                "updated_at": str(updated_at)
+            })
+
+        return conversations
+
+    except Exception as e:
+        print(f"Error retrieving conversations: {e}")
+        return None
+
+    finally:
+        cursor.close()
+        conn.close()
+
+
+if __name__ == "__main__":
+    # Step 1: Create the table
+    #deleting the table 
+    print("Step none: Deleting table...")
+    delete_table_by_name("stored_convoId_data")  # Deletes the 'stored_convoId_data' table
+    print("Table 'stored_convoId_data' deleted successfully or does not exist.")
+
+    print("Step 1: Creating table...")
+    if create_table():
+        # Step 2: Insert data with is_saved=False
+        print("\nStep 2: Inserting data with is_saved=False...")
+        insert_convo_data("12345", data, is_saved=False)
+        
+        # Step 3: Insert another conversation with is_saved=True
+        print("\nStep 3: Inserting another conversation with is_saved=True...")
+        insert_convo_data("67890", data, is_saved=True)
+        
+        # Step 4: Retrieve all data by convo_id
+        print("\n" + "="*60)
+        print("Step 4: Retrieving all data for convo_id='12345'...")
+        print("="*60)
+        convo_data = get_convo_data("12345")
+        if convo_data:
+            print(json.dumps(convo_data, indent=2))
+        
+        # Step 5: Update is_saved status
+        print("\n" + "="*60)
+        print("Step 5: Updating is_saved status to True for convo_id='12345'...")
+        print("="*60)
+        update_saved_status("12345", is_saved=True)
+        
+        # Step 6: Retrieve saved conversations for specific convo_id
+        print("\n" + "="*60)
+        print("Step 6: Retrieving saved conversations for convo_id='12345'...")
+        print("="*60)
+        saved_convo = get_convo_data_by_saved_status("12345", is_saved=True)
+        if saved_convo:
+            print(json.dumps(saved_convo, indent=2))
+        
+        # Step 7: Retrieve all saved conversations
+        print("\n" + "="*60)
+        print("Step 7: Retrieving ALL saved conversations...")
+        print("="*60)
+        all_saved = get_all_saved_conversations()
+        if all_saved:
+            print(f"Found {len(all_saved)} saved conversation(s):")
+            print(json.dumps(all_saved, indent=2))
+    else:
+        print("Failed to create table. Exiting...")
+
+    # Step 8: Retrieve all conversations
+       # Get all saved conversations
+        print("\n" + "="*60)
+        saved_convos = get_all_convo_data_filtered(is_saved=True)
+        if saved_convos:
+            print(f"Found {len(saved_convos)} saved conversation(s):")
+            print(json.dumps(saved_convos, indent=2))
+        print("="*60)
+        # Get all unsaved conversations
+        unsaved_convos = get_all_convo_data_filtered(is_saved=False)
+        if unsaved_convos:
+            print(f"Found {len(unsaved_convos)} unsaved conversation(s):")
+            print(json.dumps(unsaved_convos, indent=2))
+
+        print("="*60)
+
+        # Get all conversations (both saved and unsaved)
+        all_convos = get_all_convo_data_filtered(is_saved=None)
+        if all_convos:
+            print(f"Found {len(all_convos)} conversation(s):")
+            print(json.dumps(all_convos, indent=2))

DB_store_backup/stored_convoId_data.py

@@ -0,0 +1,423 @@
+import psycopg2
+from psycopg2 import sql
+import json
+
+# PostgreSQL credentials
+PGHOST = 'ep-steep-dream-adqtvjel-pooler.c-2.us-east-1.aws.neon.tech'
+PGDATABASE = 'neondb'
+PGUSER = 'neondb_owner'
+PGPASSWORD = 'npg_Qq0B1uWRXavx'
+PGSSLMODE = 'require'
+
+# Function to connect to the PostgreSQL database
+def connect_to_db():
+    try:
+        conn = psycopg2.connect(
+            host=PGHOST,
+            database=PGDATABASE,
+            user=PGUSER,
+            password=PGPASSWORD,
+            sslmode=PGSSLMODE
+        )
+        return conn
+    except Exception as e:
+        print(f"Error connecting to PostgreSQL database: {e}")
+        return None
+    
+# Function to delete the table 
+def delete_table_by_name(table_name: str):
+    conn = connect_to_db()
+    if not conn:
+        print("Failed to connect to the database")
+        return
+
+    cursor = conn.cursor()
+
+    # Create the DROP TABLE query dynamically using the table_name argument
+    drop_table_query = sql.SQL("DROP TABLE IF EXISTS {}").format(sql.Identifier(table_name))
+
+    try:
+        cursor.execute(drop_table_query)
+        conn.commit()
+        print(f"Table '{table_name}' deleted successfully.")
+    except Exception as e:
+        conn.rollback()
+        print(f"Error deleting table '{table_name}': {e}")
+    finally:
+        cursor.close()
+        conn.close()
+
+
+
+# Function to create the table if it doesn't exist
+def create_table():
+    conn = connect_to_db()
+    if not conn:
+        return False
+
+    cursor = conn.cursor()
+
+    create_table_query = """
+        CREATE TABLE IF NOT EXISTS "stored_convoId_data" (
+            id SERIAL PRIMARY KEY,
+            convo_id VARCHAR(255) NOT NULL,
+            user_query JSONB NOT NULL,
+            response JSONB NOT NULL,
+            file_metadata JSONB,
+            is_saved BOOLEAN DEFAULT FALSE,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+    """
+
+    # Create index for faster queries
+    create_index_query = """
+        CREATE INDEX IF NOT EXISTS idx_convo_id_is_saved 
+        ON "stored_convoId_data" (convo_id, is_saved);
+    """
+
+    try:
+        cursor.execute(create_table_query)
+        cursor.execute(create_index_query)
+        conn.commit()
+        print("Table 'stored_convoId_data' created successfully or already exists.")
+        return True
+    except Exception as e:
+        print(f"Error creating table: {e}")
+        conn.rollback()
+        return False
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to insert the data into PostgreSQL table
+def insert_convo_data(convo_id, data, is_saved=False):
+    conn = connect_to_db()
+    if not conn:
+        return
+
+    cursor = conn.cursor()
+
+    insert_query = sql.SQL("""
+        INSERT INTO "stored_convoId_data" (convo_id, user_query, response, file_metadata, is_saved)
+        VALUES (%s, %s, %s, %s, %s)
+    """)
+
+    # Prepare data to insert (use json.dumps for JSONB columns)
+    user_query = json.dumps(data["user_query"])
+    response = json.dumps(data["response"])
+    file_metadata = json.dumps(data["response"]["artifacts"])
+
+    try:
+        cursor.execute(insert_query, (convo_id, user_query, response, file_metadata, is_saved))
+        conn.commit()
+        print(f"Data for convo_id {convo_id} inserted successfully with is_saved={is_saved}.")
+    except Exception as e:
+        print(f"Error inserting data: {e}")
+        conn.rollback()
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to update the is_saved flag
+def update_saved_status(convo_id, is_saved=True):
+    conn = connect_to_db()
+    if not conn:
+        return False
+
+    cursor = conn.cursor()
+
+    update_query = """
+        UPDATE "stored_convoId_data" 
+        SET is_saved = %s, updated_at = CURRENT_TIMESTAMP
+        WHERE convo_id = %s
+    """
+
+    try:
+        cursor.execute(update_query, (is_saved, convo_id))
+        conn.commit()
+        if cursor.rowcount > 0:
+            print(f"Updated {cursor.rowcount} record(s) for convo_id {convo_id} to is_saved={is_saved}.")
+            return True
+        else:
+            print(f"No records found for convo_id {convo_id}.")
+            return False
+    except Exception as e:
+        print(f"Error updating saved status: {e}")
+        conn.rollback()
+        return False
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to retrieve data by convo_id only
+def get_convo_data(convo_id):
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE convo_id = %s
+        ORDER BY created_at DESC
+    """
+
+    try:
+        cursor.execute(select_query, (convo_id,))
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved, created_at, updated_at = result
+                
+                # JSONB columns are already returned as dict/list by psycopg2
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print(f"No data found for convo_id: {convo_id}")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving data: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to retrieve data by convo_id and is_saved status
+def get_convo_data_by_saved_status(convo_id, is_saved=True):
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE convo_id = %s AND is_saved = %s
+        ORDER BY created_at DESC
+    """
+
+    try:
+        cursor.execute(select_query, (convo_id, is_saved))
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved_flag, created_at, updated_at = result
+                
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved_flag,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print(f"No data found for convo_id: {convo_id} with is_saved={is_saved}")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving data: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Function to get all saved conversations (across all convo_ids)
+def get_all_saved_conversations():
+    conn = connect_to_db()
+    if not conn:
+        return None
+
+    cursor = conn.cursor()
+
+    select_query = """
+        SELECT convo_id, user_query, response, file_metadata, is_saved, created_at, updated_at
+        FROM "stored_convoId_data" 
+        WHERE is_saved = TRUE
+        ORDER BY updated_at DESC
+    """
+
+    try:
+        cursor.execute(select_query)
+        results = cursor.fetchall()
+        
+        if results:
+            conversations = []
+            for result in results:
+                stored_convo_id, user_query_json, response_json, file_metadata_json, is_saved, created_at, updated_at = result
+                
+                user_query = user_query_json if isinstance(user_query_json, dict) else json.loads(user_query_json)
+                response = response_json if isinstance(response_json, dict) else json.loads(response_json)
+                file_metadata = file_metadata_json if isinstance(file_metadata_json, (dict, list)) else json.loads(file_metadata_json)
+                
+                conversations.append({
+                    "convo_id": stored_convo_id,
+                    "user_query": user_query,
+                    "response": response,
+                    "file_metadata": file_metadata,
+                    "is_saved": is_saved,
+                    "created_at": str(created_at),
+                    "updated_at": str(updated_at)
+                })
+            
+            return conversations
+        else:
+            print("No saved conversations found.")
+            return None
+
+    except Exception as e:
+        print(f"Error retrieving saved conversations: {e}")
+        return None
+    finally:
+        cursor.close()
+        conn.close()
+
+# Sample data
+data = {
+  "user_query": {
+    "query_id": "12345",
+    "text": "What is the weather like today?",
+    "user_metadata": {
+      "location": "New York",
+      "language": "en"
+    }
+  },
+  "response": {
+    "text": "The weather today in New York is sunny and 75°F. Would you like to see a detailed report or a forecast?",
+    "status": "success",
+    "response_time": "2025-10-13T08:10:00Z",
+    "duration": "2s",
+    "artifacts": [
+      {
+        "artifact_id": "artifact_1",
+        "file_id": "file_1",
+        "file_name": "weather_report.pdf",
+        "file_type": "pdf",
+        "file_size": 1024,
+        "file_url": "path_to_file",
+        "upload_timestamp": "2025-10-13T08:00:00Z",
+        "metadata": {
+          "created_by": "system",
+          "associated_query_id": "12345",
+          "associated_session_id": "session_001",
+          "artifact_timestamp": "2025-10-13T08:10:00Z"
+        }
+      },
+      {
+        "artifact_id": "artifact_2",
+        "file_id": "file_2",
+        "file_name": "weather_forecast_image.jpg",
+        "file_type": "jpg",
+        "file_size": 2048,
+        "file_url": "path_to_image",
+        "upload_timestamp": "2025-10-13T08:05:00Z",
+        "metadata": {
+          "created_by": "system",
+          "associated_query_id": "12345",
+          "associated_session_id": "session_001",
+          "artifact_timestamp": "2025-10-13T08:10:00Z"
+        }
+      }
+    ]
+  },
+  "metadata": {
+    "query_response_metadata": {
+      "response_time": "2025-10-13T08:10:00Z",
+      "response_status": "success",
+      "response_duration": "2s"
+    },
+    "file_metadata": [
+      {
+        "file_id": "file_1",
+        "file_name": "weather_report.pdf",
+        "file_size": 1024,
+        "upload_timestamp": "2025-10-13T08:00:00Z"
+      },
+      {
+        "file_id": "file_2",
+        "file_name": "weather_forecast_image.jpg",
+        "file_size": 2048,
+        "upload_timestamp": "2025-10-13T08:05:00Z"
+      }
+    ]
+  }
+}
+
+if __name__ == "__main__":
+    # Step 1: Create the table
+    #deleting the table 
+    # print("Step none: Deleting table...")
+    # delete_table_by_name("stored_convoId_data")  # Deletes the 'stored_convoId_data' table
+    print("Table 'stored_convoId_data' deleted successfully or does not exist.")
+
+    print("Step 1: Creating table...")
+    if create_table():
+        # Step 2: Insert data with is_saved=False
+        print("\nStep 2: Inserting data with is_saved=False...")
+        insert_convo_data("12345", data, is_saved=False)
+        
+        # Step 3: Insert another conversation with is_saved=True
+        print("\nStep 3: Inserting another conversation with is_saved=True...")
+        insert_convo_data("67890", data, is_saved=True)
+        
+        # Step 4: Retrieve all data by convo_id
+        print("\n" + "="*60)
+        print("Step 4: Retrieving all data for convo_id='12345'...")
+        print("="*60)
+        convo_data = get_convo_data("12345")
+        if convo_data:
+            print(json.dumps(convo_data, indent=2))
+        
+        # Step 5: Update is_saved status
+        print("\n" + "="*60)
+        print("Step 5: Updating is_saved status to True for convo_id='12345'...")
+        print("="*60)
+        update_saved_status("12345", is_saved=True)
+        
+        # Step 6: Retrieve saved conversations for specific convo_id
+        print("\n" + "="*60)
+        print("Step 6: Retrieving saved conversations for convo_id='12345'...")
+        print("="*60)
+        saved_convo = get_convo_data_by_saved_status("12345", is_saved=True)
+        if saved_convo:
+            print(json.dumps(saved_convo, indent=2))
+        
+        # Step 7: Retrieve all saved conversations
+        print("\n" + "="*60)
+        print("Step 7: Retrieving ALL saved conversations...")
+        print("="*60)
+        all_saved = get_all_saved_conversations()
+        if all_saved:
+            print(f"Found {len(all_saved)} saved conversation(s):")
+            print(json.dumps(all_saved, indent=2))
+    else:
+        print("Failed to create table. Exiting...")

Dockerfile

@@ -0,0 +1,74 @@
+FROM python:3.10-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements file
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application code
+COPY . .
+
+# Create all necessary directories with proper permissions
+RUN mkdir -p /app/tmp \
+             /app/.config/matplotlib \
+             /app/.cache \
+             /app/.local && \
+    chmod -R 777 /app/tmp /app/.config /app/.cache /app/.local
+
+# Set comprehensive environment variables
+ENV TMPDIR=/app/tmp \
+    TEMP=/app/tmp \
+    TMP=/app/tmp \
+    MPLCONFIGDIR=/app/.config/matplotlib \
+    XDG_CACHE_HOME=/app/.cache \
+    XDG_CONFIG_HOME=/app/.config \
+    HOME=/app \
+    MPLBACKEND=Agg \
+    PYTHONUNBUFFERED=1
+
+# Expose the port FastAPI will run on
+EXPOSE 7860
+
+# Command to run the FastAPI application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--timeout-keep-alive", "120"]
+
+
+# # Builder stage
+# FROM python:3.10-slim AS builder
+# WORKDIR /app
+# COPY requirements.txt .
+# RUN apt-get update && apt-get install -y gcc g++ \
+#     && pip install --no-cache-dir -r requirements.txt \
+#     && apt-get purge -y gcc g++ \
+#     && apt-get autoremove -y \
+#     && rm -rf /var/lib/apt/lists/*
+
+# # Runtime stage
+# FROM python:3.10-slim
+# WORKDIR /app
+# COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages
+# COPY . .
+# # Create TMP directories
+# RUN mkdir -p /app/tmp /app/.config/matplotlib /app/.cache /app/.local && \
+#     chmod -R 777 /app/tmp /app/.config /app/.cache /app/.local
+# ENV TMPDIR=/app/tmp \
+#     TEMP=/app/tmp \
+#     TMP=/app/tmp \
+#     MPLCONFIGDIR=/app/.config/matplotlib \
+#     XDG_CACHE_HOME=/app/.cache \
+#     XDG_CONFIG_HOME=/app/.config \
+#     HOME=/app \
+#     MPLBACKEND=Agg \
+#     PYTHONUNBUFFERED=1
+# EXPOSE 7860
+# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1", "--timeout-keep-alive", "120"]

Redis/rough.py

@@ -0,0 +1,132 @@
+import asyncio
+import os
+from agents import Agent, Runner
+from agents.extensions.memory import RedisSession
+import dotenv
+import redis.asyncio as aioredis
+
+dotenv.load_dotenv()
+Redis_url = os.getenv("REDIS_URL")
+
+
+#===============transformers================
+from transformers import pipeline
+import re
+from datetime import datetime
+import json
+import requests
+
+# Initialize summarization model
+try:
+    print("🚀 Loading BART model for summarization...")
+    summarizer = pipeline("summarization", model="facebook/bart-large-cnn", tokenizer="facebook/bart-large-cnn",framework="pt")
+except Exception as e:
+    print(f"Error loading BART model: {e}")
+    summarizer = None
+
+print("✅ Summarization pipeline ready!")
+
+
+
+
+# ---------- Utility Functions ----------
+
+async def get_sessions(user_login_id: str):
+    """
+    Get all session IDs for a given user (based on key prefix).
+    """
+    redis = await aioredis.from_url(Redis_url)
+    pattern = f"{user_login_id}:*"
+    keys = await redis.keys(pattern)
+    sessions = [key.decode().replace(f"{user_login_id}:", "") for key in keys]
+    await redis.close()
+    return sessions
+
+
+async def get_session_history(user_login_id: str, session_id: str):
+    """
+    Retrieve chat history for a given user's session.
+    """
+    try:
+        session = RedisSession.from_url(
+            session_id,
+            url=Redis_url,
+            key_prefix=f"{user_login_id}:",
+        )
+        if not await session.ping():
+            raise Exception("Redis connection failed")
+
+        items = await session.get_items()
+        history = [
+            {"role": msg.get("role", "unknown"), "content": msg.get("content", "")}
+            for msg in items
+        ]
+        await session.close()
+        return history
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+async def delete_session(user_login_id: str, session_id: str):
+    """
+    Delete a specific session for a given user.
+    """
+    try:
+        session = RedisSession.from_url(
+            session_id,
+            url=Redis_url,
+            key_prefix=f"{user_login_id}:",
+        )
+        if not await session.ping():
+            raise Exception("Redis connection failed")
+
+        await session.clear_session()
+        await session.close()
+        return {"status": "success", "message": f"Session {session_id} deleted"}
+
+    except Exception as e:
+        return {"status": "error", "message": str(e)}
+
+
+# ---------- Example Usage ----------
+
+async def main_demo():
+    user_id = "vatsav_user2"
+    session_id = ":uuid_12345"
+
+    print("Creating session...")
+    session = RedisSession.from_url(
+        session_id,
+        url=Redis_url,
+        key_prefix=f"{user_id}:",
+    )
+
+    agent = Agent(name="Assistant", instructions="Be concise.")
+    await Runner.run(agent, "Hello!", session=session)
+    await Runner.run(agent, "How are you?", session=session)
+    await session.close()
+
+    print("\n--- All Sessions ---")
+    print(await get_sessions(user_id))
+    print("lenth of the sessions: ", len(await get_sessions(user_id)) or 0)
+
+    print("\n--- Session History ---")
+    
+    history = await get_session_history(user_id, session_id)
+    print("lenght of the history: ", len(history) or 0)
+    print(history)
+    print("\nHistoryends=======================:")
+    for msg in history:
+        print(f"{msg['role']}: {msg['content']}")
+       
+
+
+    
+
+    print("\n--- Delete Session ---")
+    print(await delete_session(user_id, session_id))
+
+
+if __name__ == "__main__":
+    asyncio.run(main_demo())

Redis/sessions.py

@@ -0,0 +1,121 @@
+import os
+import asyncio
+import dotenv
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+import redis.asyncio as aioredis
+from agents import Agent, Runner
+from agents.extensions.memory import RedisSession
+# from transformers import pipeline
+from fastapi import APIRouter
+
+dotenv.load_dotenv()
+Redis_url = os.getenv("REDIS_URL")
+
+
+Redis_session_router = APIRouter(prefix="/Redis", tags=["Redis_Agent_Memory_Management"])
+# ============ Transformers ============ #
+# try:
+#     print("🚀 Loading summarization model (BART)...")
+#     summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
+#     print("✅ Model ready!")
+# except Exception as e:
+#     print(f"⚠️ Error loading model: {e}")
+#     summarizer = None
+
+
+# ---------- Core Utility Functions ---------- #
+async def get_sessions(user_login_id: str):
+    redis = await aioredis.from_url(Redis_url)
+    pattern = f"{user_login_id}:*"
+    keys = await redis.keys(pattern)
+    sessions = [key.decode().replace(f"{user_login_id}:", "") for key in keys]
+    await redis.close()
+    return sessions
+
+
+async def get_session_history(user_login_id: str, session_id: str):
+    """
+    Retrieve chat history for a given user's session.
+    """
+    try:
+        session = RedisSession.from_url(
+            session_id,
+            url=Redis_url,
+            key_prefix=f"{user_login_id}:",
+        )
+        if not await session.ping():
+            raise Exception("Redis connection failed")
+
+        items = await session.get_items()
+        history = [
+            {"role": msg.get("role", "unknown"), "content": msg.get("content", "")}
+            for msg in items
+        ]
+        await session.close()
+        return history
+
+    except Exception as e:
+        return {"error": str(e)}
+
+async def delete_session(user_login_id: str, session_id: str):
+    session = RedisSession.from_url(session_id, url=Redis_url, key_prefix=f"{user_login_id}:")
+    if not await session.ping():
+        raise HTTPException(status_code=500, detail="Redis connection failed")
+    await session.clear_session()
+    await session.close()
+    return {"status": "success", "message": f"Session {session_id} deleted"}
+
+
+# ---------- API Endpoints ---------- #
+
+@Redis_session_router.get("/sessions/{user_login_id}")
+async def api_get_sessions(user_login_id: str):
+    """List all sessions for a given user."""
+    try:
+        sessions = await get_sessions(user_login_id)
+        return {"user": user_login_id, "sessions": sessions, "count": len(sessions)}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@Redis_session_router.get("/sessions/{user_login_id}/{session_id}")
+async def api_get_session_history(user_login_id: str, session_id: str):
+    """Get the chat history of a session. Optionally summarize with ?summarize=true."""
+    try:
+        history = await get_session_history(user_login_id, session_id)
+        if not history:
+            raise HTTPException(status_code=404, detail="Session history not found")
+
+        result = {"user": user_login_id, "session_id": session_id, "history": history}
+
+        
+        text = " ".join([msg["content"] for msg in history if msg["role"] == "user"])
+        summary = summarizer(text, max_length=130, min_length=30, do_sample=False)[0]["summary_text"]
+        result["summary"] = summary
+        return result
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@Redis_session_router.delete("/sessions/{user_login_id}/{session_id}")
+async def api_delete_session(user_login_id: str, session_id: str):
+    """Delete a session."""
+    try:
+        result = await delete_session(user_login_id, session_id)
+        return JSONResponse(content=result)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# ---------- Demo route (optional) ---------- #
+@Redis_session_router.get("/")
+async def root():
+    return {"message": "Redis Session Manager API running 🧠"}
+
+
+# ---------- Local testing ---------- #
+# if __name__ == "__main__":
+#     import uvicorn
+#     uvicorn.run("main:Redis_session_router", host="0.0.0.0", port=8000, reload=True)

Redis/sessions_new.py

@@ -0,0 +1,203 @@
+from fastapi import APIRouter, HTTPException
+
+import json
+from pydantic import BaseModel
+from typing import Optional, List, Dict
+
+import redis, os, uuid, json
+from fastapi import HTTPException
+from datetime import datetime
+from dotenv import load_dotenv
+from fastapi import APIRouter
+import dotenv
+dotenv.load_dotenv()
+
+
+
+load_dotenv()
+Redis_url = os.getenv("REDIS_URL")
+
+
+_new_new = APIRouter(prefix="/redis_session_new", tags=["redis_session_new"])
+
+
+class ChatRequest(BaseModel):
+    user_login_id: str
+    session_id: Optional[str] = None
+    query: str
+    org_id: Optional[str] = None
+    metadata: Optional[Dict] = None
+
+class ChatResponse(BaseModel):
+    session_id: str
+    user_message: str
+    assistant_response: str
+    is_new_session: bool
+    session_title: str
+    timestamp: str
+
+class MessageResponse(BaseModel):
+    message_id: str
+    role: str
+    content: str
+    timestamp: str
+
+class ChatHistoryResponse(BaseModel):
+    session_id: str
+    title: str
+    created_at: str
+    message_count: int
+    messages: List[MessageResponse]
+
+redis_session_route_new = APIRouter(prefix="/main_chatbot", tags=["Redis_session_main_chatbot"])
+
+
+
+
+
+def get_redis_client():
+    try:
+        REDIS_URL = os.getenv("REDIS_URL")
+        if REDIS_URL:
+            return redis.from_url(REDIS_URL, decode_responses=True)
+        return redis.StrictRedis(
+            host=os.getenv("REDIS_HOST", "localhost"),
+            port=int(os.getenv("REDIS_PORT", 6379)),
+            password=os.getenv("REDIS_PASSWORD"),
+            decode_responses=True
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Redis connection error: {e}")
+
+redis_client = get_redis_client()
+
+def create_session(user_login_id, org_id=None, metadata=None):
+    session_id = str(uuid.uuid4())
+    data = {
+        "session_id": session_id,
+        "user_login_id": user_login_id,
+        "org_id": org_id,
+        "created_at": datetime.now().isoformat(),
+        "title": "New Chat",
+        "message_count": 0,
+        "metadata": metadata or {}
+    }
+    redis_client.setex(f"session:{user_login_id}:{session_id}", 86400, json.dumps(data))
+    redis_client.setex(f"messages:{user_login_id}:{session_id}", 86400, json.dumps([]))
+    return data
+
+def get_session(user_login_id, session_id):
+    key = f"session:{user_login_id}:{session_id}"
+    data = redis_client.get(key)
+    if not data:
+        raise HTTPException(status_code=404, detail="Session not found")
+    return json.loads(data)
+
+def add_message(user_login_id, session_id, role, content):
+    msg = {
+        "message_id": str(uuid.uuid4()),
+        "role": role,
+        "content": content,
+        "timestamp": datetime.now().isoformat()
+    }
+    key = f"messages:{user_login_id}:{session_id}"
+    messages = json.loads(redis_client.get(key) or "[]")
+    messages.append(msg)
+    redis_client.setex(key, 86400, json.dumps(messages))
+    session = get_session(user_login_id, session_id)
+    session["message_count"] = len(messages)
+    redis_client.setex(f"session:{user_login_id}:{session_id}", 86400, json.dumps(session))
+    return msg["message_id"]
+
+def get_message_history(user_login_id, session_id, limit=None):
+    key = f"messages:{user_login_id}:{session_id}"
+    data = redis_client.get(key)
+    if not data:
+        return []
+    messages = json.loads(data)
+    return messages[-limit:] if limit else messages
+
+def generate_session_title(user_login_id, session_id):
+    messages = get_message_history(user_login_id, session_id)
+    first = next((m["content"] for m in messages if m["role"] == "user"), "")
+    title = " ".join(first.split()[:6]) + ("..." if len(first.split()) > 6 else "")
+    session = get_session(user_login_id, session_id)
+    session["title"] = title or "New Chat"
+    redis_client.setex(f"session:{user_login_id}:{session_id}", 86400, json.dumps(session))
+    return title or "New Chat"
+
+def update_session_title_if_needed(user_login_id, session_id, is_new):
+    session = get_session(user_login_id, session_id)
+    if is_new or session.get("title") == "New Chat":
+        generate_session_title(user_login_id, session_id)
+
+def format_conversation_context(messages, max_messages=5):
+    if not messages:
+        return ""
+    return "\n".join(f"{m['role']}: {m['content']}" for m in messages[-max_messages:])
+
+@redis_session_route_new.get("/history/{user_login_id}/{session_id}", response_model=ChatHistoryResponse)
+async def get_chat_history(user_login_id: str, session_id: str):
+    session_data = get_session(user_login_id, session_id)
+    messages = get_message_history(user_login_id, session_id)
+
+    message_responses = [
+        MessageResponse(**msg) for msg in messages
+    ]
+
+    return ChatHistoryResponse(
+        session_id=session_id,
+        title=session_data.get("title", "New Chat"),
+        created_at=session_data.get("created_at"),
+        message_count=len(messages),
+        messages=message_responses
+    )
+
+@redis_session_route_new.get("/sessions/{user_login_id}")
+async def get_user_sessions(user_login_id: str):
+    sessions = []
+    pattern = f"session:{user_login_id}:*"
+    for key in redis_client.scan_iter(match=pattern):
+        session_data = redis_client.get(key)
+        if session_data:
+            sessions.append(json.loads(session_data))
+    sessions.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+    return {
+        "user_login_id": user_login_id,
+        "total_sessions": len(sessions),
+        "sessions": sessions
+    }
+
+@redis_session_route_new.delete("/sessions/{user_login_id}/{session_id}")
+async def delete_session(user_login_id: str, session_id: str):
+    get_session(user_login_id, session_id)  # Validate existence
+    redis_client.delete(f"session:{user_login_id}:{session_id}")
+    redis_client.delete(f"messages:{user_login_id}:{session_id}")
+    return {"message": "Session deleted", "session_id": session_id}
+
+@redis_session_route_new.get("/health")
+async def health():
+    try:
+        redis_client.ping()
+        status = "connected"
+        total_sessions = len(list(redis_client.scan_iter(match="session:*")))
+    except:
+        status = "disconnected"
+        total_sessions = 0
+    return {
+        "status": "ok",
+        "redis_status": status,
+        "total_sessions": total_sessions,
+        "session_ttl": "24 hours"
+    }
+
+@redis_session_route_new.put("/sessions/{user_login_id}/{session_id}/title")
+async def update_session_title(user_login_id: str, session_id: str, title: str):
+    key = f"session:{user_login_id}:{session_id}"
+    session_data = redis_client.get(key)
+    if not session_data:
+        raise HTTPException(status_code=404, detail="Session not found")
+    session = json.loads(session_data)
+    session["title"] = title
+    redis_client.setex(key, 86400, json.dumps(session))
+    return {"message": "Title updated"}

Redis/sessions_old.py

@@ -0,0 +1,598 @@
+from fastapi import FastAPI, HTTPException, Query as QueryParam, Request
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict, Any
+import os
+import json
+import redis
+import uuid
+from datetime import datetime
+from dotenv import load_dotenv
+from fastapi import APIRouter
+
+load_dotenv()
+
+
+
+# Redis_session_router_old = FastAPI(title="Redis Session Management API")
+Redis_session_router_old = APIRouter(prefix="/Redis", tags=["Redis_Management_old"])
+
+# ==================== CONFIGURATION ====================
+
+# Redis Configuration
+REDIS_URL = os.getenv("REDIS_URL")
+REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
+REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
+
+# ==================== REDIS CLIENT INITIALIZATION ====================
+
+def get_redis_client():
+    """Initialize Redis client with fallback to local Redis"""
+    try:
+        if REDIS_URL:
+            # Use deployed Redis URL
+            redis_client = redis.from_url(
+                REDIS_URL,
+                decode_responses=True,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            # Test connection
+            redis_client.ping()
+            print(f"✅ Connected to deployed Redis: {REDIS_URL}")
+            return redis_client
+        else:
+            # Use local Redis
+            redis_client = redis.StrictRedis(
+                host=REDIS_HOST,
+                port=REDIS_PORT,
+                password=REDIS_PASSWORD,
+                decode_responses=True,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            # Test connection
+            redis_client.ping()
+            print(f"✅ Connected to local Redis: {REDIS_HOST}:{REDIS_PORT}")
+            return redis_client
+    except Exception as e:
+        print(f"❌ Redis connection failed: {e}")
+        raise HTTPException(status_code=500, detail=f"Redis connection failed: {str(e)}")
+
+# Initialize Redis client
+# redis_client = get_redis_client()
+
+# ==================== PYDANTIC MODELS ====================
+
+class SessionResponse(BaseModel):
+    session_id: str
+    userLoginId: int
+    orgId: int
+    created_at: str
+    status: str
+    title: Optional[str] = "New Chat"
+
+class MessageResponse(BaseModel):
+    message_id: str
+    session_id: str
+    role: str  # "user" or "assistant"
+    message: str
+    timestamp: str
+
+class ChatHistoryResponse(BaseModel):
+    session_id: str
+    messages: List[MessageResponse]
+    total_messages: int
+
+class UpdateSessionTitleRequest(BaseModel):
+    new_title: str
+
+class CreateSessionRequest(BaseModel):
+    userLoginId: int
+    orgId: int
+    auth_token: str
+
+class AddMessageRequest(BaseModel):
+    session_id: str
+    role: str
+    message: str
+
+# ==================== SESSION MANAGEMENT FUNCTIONS ====================
+
+def create_session(userLoginId: int, orgId: int, auth_token: str) -> dict:
+    """Create a new chat session"""
+    session_id = str(uuid.uuid4())
+    session_data = {
+        "session_id": session_id,
+        "userLoginId": userLoginId,
+        "orgId": orgId,
+        "auth_token": auth_token,
+        "created_at": datetime.now().isoformat(),
+        "status": "active",
+        "title": "New Chat"
+    }
+    
+    # Store session in Redis with 24 hour TTL
+    redis_client.setex(
+        f"session:{session_id}", 
+        86400,  # 24 hours
+        json.dumps(session_data)
+    )
+    
+    # Initialize empty chat history
+    redis_client.setex(
+        f"chat:{session_id}",
+        86400,  # 24 hours
+        json.dumps([])
+    )
+    
+    # Initialize conversation memory
+    redis_client.setex(
+        f"memory:{session_id}",
+        86400,  # 24 hours
+        json.dumps([])
+    )
+    
+    return session_data
+
+def get_session(session_id: str) -> dict:
+    """Get session data from Redis"""
+    session_data = redis_client.get(f"session:{session_id}")
+    if not session_data:
+        raise HTTPException(status_code=404, detail="Session not found or expired")
+    return json.loads(session_data)
+
+def add_message_to_session(session_id: str, role: str, message: str) -> str:
+    """Add message to session chat history"""
+    message_id = str(uuid.uuid4())
+    message_data = {
+        "message_id": message_id,
+        "session_id": session_id,
+        "role": role,
+        "message": message,
+        "timestamp": datetime.now().isoformat()
+    }
+    
+    # Get current chat history
+    chat_history = redis_client.get(f"chat:{session_id}")
+    if chat_history:
+        messages = json.loads(chat_history)
+    else:
+        messages = []
+    
+    # Add new message
+    messages.Redis_session_routerend(message_data)
+    
+    # Update chat history in Redis with extended TTL
+    redis_client.setex(
+        f"chat:{session_id}",
+        86400,  # 24 hours
+        json.dumps(messages)
+    )
+    
+    return message_id
+
+def get_session_memory(session_id: str) -> List[Dict]:
+    """Get conversation memory for session"""
+    memory_data = redis_client.get(f"memory:{session_id}")
+    if memory_data:
+        return json.loads(memory_data)
+    return []
+
+def update_session_memory(session_id: str, messages: List[Dict]):
+    """Update conversation memory for session"""
+    redis_client.setex(
+        f"memory:{session_id}",
+        86400,  # 24 hours
+        json.dumps(messages)
+    )
+
+def generate_session_title(session_id: str) -> str:
+    """Generate a title for the session based on chat history"""
+    try:
+        # Check session
+        session_data = redis_client.get(f"session:{session_id}")
+        if session_data:
+            session = json.loads(session_data)
+            if "user_title" in session:
+                # Don't override user-defined titles
+                return session["user_title"]
+
+        # Get chat history
+        chat_data = redis_client.get(f"chat:{session_id}")
+        if not chat_data:
+            return "New Chat"
+
+        messages = json.loads(chat_data)
+        if not messages:
+            return "New Chat"
+
+        # Get first user message
+        first_user_message = next(
+            (msg["message"] for msg in messages if msg["role"] == "user"), None
+        )
+
+        if not first_user_message:
+            return "New Chat"
+
+        # Create a simple title from first message
+        words = first_user_message.split()[:6]
+        title = " ".join(words) + ("..." if len(first_user_message.split()) > 6 else "")
+
+        # Save to session
+        if session_data:
+            session["generated_title"] = title
+            if not session.get("user_title"):
+                session["title"] = title
+            redis_client.setex(f"session:{session_id}", 86400, json.dumps(session))
+
+        return title
+
+    except Exception as e:
+        print(f"Error in generate_session_title: {e}")
+        return "New Chat"
+
+def update_session_title(session_id: str):
+    """Update session title after first message"""
+    try:
+        # Get session data
+        session_data = redis_client.get(f"session:{session_id}")
+        if not session_data:
+            return
+        
+        session = json.loads(session_data)
+        
+        # Only update if current title is "New Chat"
+        if session.get("title", "New Chat") == "New Chat":
+            new_title = generate_session_title(session_id)
+            session["title"] = new_title
+            
+            # Update session in Redis
+            redis_client.setex(
+                f"session:{session_id}",
+                86400,  # 24 hours
+                json.dumps(session)
+            )
+            
+    except Exception as e:
+        print(f"Error updating session title: {e}")
+        pass
+
+def get_user_sessions(userLoginId: int) -> List[dict]:
+    """Get all sessions for a user with generated titles"""
+    sessions = []
+    # Scan for all session keys
+    for key in redis_client.scan_iter(match="session:*"):
+        session_data = redis_client.get(key)
+        if session_data:
+            session = json.loads(session_data)
+            if session["userLoginId"] == userLoginId:
+                # Generate title based on chat history
+                session["title"] = generate_session_title(session["session_id"])
+                sessions.Redis_session_routerend(session)
+                
+    
+    # Sort sessions by created_at (most recent first)
+    sessions.sort(key=lambda x: x["created_at"], reverse=True)
+
+    return sessions
+def get_user_sessions(userLoginId: int) -> List[dict]:
+    """Get all sessions for a user with generated titles"""
+    sessions = []
+    # Scan for all session keys
+    for key in redis_client.scan_iter(match="session:*"):
+        session_data = redis_client.get(key)
+        if session_data:
+            session = json.loads(session_data)
+            if session["userLoginId"] == userLoginId:
+                # Generate title based on chat history
+                session["title"] = generate_session_title(session["session_id"])
+                sessions.append(session)
+    
+    # Sort sessions by created_at (most recent first)
+    sessions.sort(key=lambda x: x["created_at"], reverse=True)
+    return sessions
+
+def delete_session(session_id: str):
+    """Delete session and associated data"""
+    # Delete session data
+    redis_client.delete(f"session:{session_id}")
+    # Delete chat history
+    redis_client.delete(f"chat:{session_id}")
+    # Delete memory
+    redis_client.delete(f"memory:{session_id}")
+
+# ==================== MIDDLEWARE ====================
+
+
+
+# ==================== API ENDPOINTS ====================
+
+@Redis_session_router_old.post("/sessions", response_model=SessionResponse)
+def create_new_session(request: CreateSessionRequest):
+    """Create a new chat session"""
+    try:
+        session_data = create_session(request.userLoginId, request.orgId, request.auth_token)
+        return SessionResponse(**session_data)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error creating session: {str(e)}")
+
+# @Redis_session_router_old.get("/sessions")
+# def list_user_sessions(userLoginId: int):
+#     """List all sessions for a user"""
+#     try:
+#         sessions = get_user_sessions(userLoginId)
+#         print(sessions)
+#         return {
+#             "userLoginId": userLoginId,
+#             "total_sessions": len(sessions),
+#             "sessions": sessions
+#         }
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error fetching sessions: {str(e)}")
+@Redis_session_router_old.get("/sessions")  
+def list_user_sessions(userLoginId: int):
+    """List all sessions for a user"""
+    try:
+        sessions = get_user_sessions(userLoginId)
+        return {
+            "userLoginId": userLoginId,
+            "total_sessions": len(sessions),
+            "sessions": sessions
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching sessions: {str(e)}")
+
+@Redis_session_router_old.get("/sessions/{session_id}")
+def get_session_details(session_id: str):
+    """Get details of a specific session"""
+    try:
+        session_data = get_session(session_id)
+        return session_data
+    except Exception as e:
+        raise HTTPException(status_code=404, detail=f"Session not found: {str(e)}")
+
+@Redis_session_router_old.delete("/sessions/{session_id}")
+def delete_user_session(session_id: str):
+    """Delete/close a session"""
+    try:
+        # Verify session exists
+        get_session(session_id)
+        
+        # Delete session
+        delete_session(session_id)
+        
+        return {
+            "message": f"Session {session_id} deleted successfully",
+            "session_id": session_id
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+@Redis_session_router_old.get("/sessions/{session_id}/history", response_model=ChatHistoryResponse)
+def get_session_history(
+    session_id: str, 
+    n: int = QueryParam(50, description="Number of recent messages to return")
+):
+    """Get chat history for a session"""
+    try:
+        # Verify session exists
+        get_session(session_id)
+        
+        # Get chat history
+        chat_data = redis_client.get(f"chat:{session_id}")
+        if not chat_data:
+            return ChatHistoryResponse(
+                session_id=session_id,
+                messages=[],
+                total_messages=0
+            )
+        
+        messages = json.loads(chat_data)
+        
+        # Get the last n messages (or all if less than n)
+        recent_messages = messages[-n:] if len(messages) > n else messages
+        
+        # Convert to MessageResponse objects
+        message_responses = [MessageResponse(**msg) for msg in recent_messages]
+        
+        return ChatHistoryResponse(
+            session_id=session_id,
+            messages=message_responses,
+            total_messages=len(messages)
+        )
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching chat history: {str(e)}")
+
+@Redis_session_router_old.post("/sessions/{session_id}/messages")
+def add_message(session_id: str, request: AddMessageRequest):
+    """Add a message to a session"""
+    try:
+        # Verify session exists
+        get_session(session_id)
+        
+        # Validate role
+        if request.role not in ["user", "assistant"]:
+            raise HTTPException(status_code=400, detail="Role must be 'user' or 'assistant'")
+        
+        # Add message
+        message_id = add_message_to_session(session_id, request.role, request.message)
+        
+        # Update title if it's the first user message
+        if request.role == "user":
+            update_session_title(session_id)
+        
+        return {
+            "message_id": message_id,
+            "session_id": session_id,
+            "role": request.role,
+            "message": request.message,
+            "timestamp": datetime.now().isoformat()
+        }
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error adding message: {str(e)}")
+
+@Redis_session_router_old.put("/sessions/{session_id}/title")
+def update_session_title_endpoint(session_id: str, request: UpdateSessionTitleRequest):
+    """Update the user-defined title of an existing session"""
+    try:
+        session_data = redis_client.get(f"session:{session_id}")
+        if not session_data:
+            raise HTTPException(status_code=404, detail="Session not found or expired")
+
+        session = json.loads(session_data)
+
+        new_title = request.new_title.strip()
+        if not new_title:
+            raise HTTPException(status_code=400, detail="New title cannot be empty")
+        if len(new_title) > 100:
+            raise HTTPException(status_code=400, detail="Title cannot exceed 100 characters")
+
+        old_title = session.get("title", "New Chat")
+        session["user_title"] = new_title
+        session["title"] = new_title
+        session["last_updated"] = datetime.now().isoformat()
+
+        redis_client.setex(f"session:{session_id}", 86400, json.dumps(session))
+
+        return {
+            "message": "Session title updated successfully",
+            "session_id": session_id,
+            "old_title": old_title,
+            "new_title": new_title
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error updating session title: {str(e)}")
+
+@Redis_session_router_old.put("/sessions/{session_id}/refresh-title")
+def refresh_session_title(session_id: str):
+    """Manually refresh/regenerate session title"""
+    try:
+        # Verify session exists
+        session_data = get_session(session_id)
+        
+        # Generate new title
+        new_title = generate_session_title(session_id)
+        
+        # Update session
+        session_data["title"] = new_title
+        redis_client.setex(
+            f"session:{session_id}",
+            86400,  # 24 hours
+            json.dumps(session_data)
+        )
+        
+        return {
+            "session_id": session_id,
+            "new_title": new_title,
+            "message": "Session title updated successfully"
+        }
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error updating session title: {str(e)}")
+
+@Redis_session_router_old.get("/sessions/{session_id}/memory")
+def get_session_memory_endpoint(session_id: str):
+    """Get conversation memory for a session"""
+    try:
+        # Verify session exists
+        get_session(session_id)
+        
+        memory = get_session_memory(session_id)
+        
+        return {
+            "session_id": session_id,
+            "memory": memory,
+            "total_items": len(memory)
+        }
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching memory: {str(e)}")
+
+# ==================== SYSTEM ENDPOINTS ====================
+
+@Redis_session_router_old.get("/redis-info")
+def redis_info():
+    """Get Redis connection information"""
+    try:
+        info = redis_client.info()
+        return {
+            "redis_connected": True,
+            "redis_version": info.get("redis_version"),
+            "used_memory": info.get("used_memory_human"),
+            "connected_clients": info.get("connected_clients"),
+            "total_keys": redis_client.dbsize()
+        }
+    except Exception as e:
+        return {
+            "redis_connected": False,
+            "error": str(e)
+        }
+
+@Redis_session_router_old.get("/health")
+def health():
+    """System health check"""
+    try:
+        redis_client.ping()
+        redis_status = "connected"
+    except:
+        redis_status = "disconnected"
+    
+    total_sessions = 0
+    if redis_status == "connected":
+        try:
+            total_sessions = len(list(redis_client.scan_iter(match="session:*")))
+        except:
+            pass
+    
+    return {
+        "status": "ok",
+        "redis_status": redis_status,
+        "session_management": "enabled",
+        "total_sessions": total_sessions,
+        "ttl": "24 hours"
+    }
+
+@Redis_session_router_old.get("/")
+def root():
+    """Root endpoint with API information"""
+    return {
+        "service": "Redis Session Management API",
+        "version": "1.0.0",
+        "endpoints": {
+            "sessions": {
+                "POST /sessions": "Create new session",
+                "GET /sessions?userLoginId={id}": "List user sessions",
+                "GET /sessions/{id}": "Get session details",
+                "DELETE /sessions/{id}": "Delete session",
+                "GET /sessions/{id}/history": "Get chat history",
+                "POST /sessions/{id}/messages": "Add message to session",
+                "PUT /sessions/{id}/title": "Update session title",
+                "PUT /sessions/{id}/refresh-title": "Refresh session title",
+                "GET /sessions/{id}/memory": "Get session memory"
+            },
+            "system": {
+                "GET /health": "Health check",
+                "GET /redis-info": "Redis connection info"
+            }
+        }
+    }
+
+# ==================== RUN SERVER ====================
+
+# if __name__ == "__main__":
+#     import uvicorn
+#     try:
+#         uvicorn.run(Redis_session_router_old, host="0.0.0.0", port=8000)
+#     except KeyboardInterrupt:
+#         print("\n🛑 Server stopped gracefully")
+#     except Exception as e:
+#         print(f"❌ Server error: {e}")

Routes/generate_report.py

@@ -0,0 +1,71 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import List, Optional
+from Routes.helpers.report_generation_helpers import return_html_report, html_to_pdf_via_api
+
+# Router initialization
+Report_Generation_Router = APIRouter(prefix="/report_generation", tags=["Report_Generation"])
+
+
+# ---------- MODELS ----------
+class PageOptions(BaseModel):
+    include_cover_page: bool
+    include_header_footer: bool
+
+
+class ReportRequest(BaseModel):
+    format_type: str  # "pdf" or "html"
+    reportname: str
+    include_citations: bool
+    success: bool
+    list_of_queries: List[str]
+    theme: str
+    pageoptions: PageOptions
+
+
+class ReportResponse(BaseModel):
+    report_generation_status: str
+    report_url: Optional[str] = None  # for PDF storage or link
+    html_content: Optional[str] = None  # for direct HTML return
+    GeGenerated_report_details: Optional[str] = None
+
+
+# ---------- ROUTES ----------
+
+@Report_Generation_Router.post("/generate_report", response_model=ReportResponse)
+async def generate_html_report(report_request: ReportRequest):
+    """
+    Generates a static HTML report using Gemini AI.
+    """
+    try:
+        html_content = return_html_report()
+        return ReportResponse(
+            report_generation_status="success",
+            html_content=html_content,
+            GeGenerated_report_details="HTML report generated successfully."
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"HTML report generation failed: {str(e)}")
+
+
+
+@Report_Generation_Router.post("/save_generated_report", response_model=ReportResponse)
+async def save_generated_report(report_request: ReportRequest):
+    """
+    Generates HTML and converts it to a PDF file via external API.
+    """
+    try:
+        html_content = return_html_report()
+        generated_report = html_to_pdf_via_api(html_content)
+
+        # Extract PDF base64 or URL from the response
+        pdf_result_info = generated_report.get("pdf_base64", "PDF generated successfully")
+
+        return ReportResponse(
+            report_generation_status="success",
+            report_url=None,  # you can replace with cloud URL if uploaded
+            GeGenerated_report_details=str(pdf_result_info)
+        )
+
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"PDF report generation failed: {str(e)}")

Routes/helpers/autovis_tool.py

@@ -0,0 +1,132 @@
+
+# from autoviz import AutoViz_Class
+# import os
+# import pandas as pd
+# import uuid
+
+# def run_autoviz(
+#     filename,
+#     sep=",",
+#     depVar="",
+#     dfte=None,
+#     header=0,
+#     verbose=2,
+#     lowess=False,
+#     chart_format="html",
+#     max_rows_analyzed=150000,
+#     max_cols_analyzed=30,
+#     save_plot_dir=None
+# ):
+#     # Generate unique directory using uuid
+#     vis_in = str(uuid.uuid4())
+#     save_plot_dir = f"./{chart_format}_{vis_in}"
+#     os.makedirs(save_plot_dir, exist_ok=True)
+
+#     # Read the CSV file to check if the path is correct
+#     df = pd.read_csv(filename)
+#     # print(df.head())
+
+#     # Run AutoViz
+#     AV = AutoViz_Class()
+#     print("Running AutoViz...")
+#     dft = AV.AutoViz(
+#         filename,
+#         sep=sep,
+#         depVar=depVar,
+#         dfte=dfte,
+#         header=header,
+#         verbose=verbose,
+#         lowess=lowess,
+#         chart_format=chart_format,
+#         max_rows_analyzed=max_rows_analyzed,
+#         max_cols_analyzed=max_cols_analyzed,
+#         save_plot_dir=save_plot_dir
+#     )
+#     print(dft)
+#     print(f"Plots saved in: {save_plot_dir}")
+#     return dft
+
+# # Example usage:
+# run_autoviz(
+#     filename=r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+# )
+
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+from autoviz import AutoViz_Class
+import os
+import pandas as pd
+import uuid
+from fastapi import APIRouter
+
+Autoviz_router = APIRouter(prefix="/autoviz", tags=["autoviz"])
+
+
+
+
+    
+
+
+class AutoVizParams(BaseModel):
+    filename: str
+    sep: str = ","
+    depVar: str = ""
+    header: int = 0
+    verbose: int = 2
+    lowess: bool = False
+    chart_format: str = "html"
+    max_rows_analyzed: int = 150000
+    max_cols_analyzed: int = 30
+
+
+global save_plot_dir
+def run_autoviz(
+    filename,
+    sep=",",
+    depVar="",
+    dfte=None,
+    header=0,
+    verbose=2,
+    lowess=False,
+    chart_format="html",
+    max_rows_analyzed=150000,
+    max_cols_analyzed=30,
+    save_plot_dir=None
+):
+    vis_in = str(uuid.uuid4())
+    save_plot_dir = f"./{chart_format}_{vis_in}"
+    os.makedirs(save_plot_dir, exist_ok=True)
+    df = pd.read_csv(filename)
+    AV = AutoViz_Class()
+    dft = AV.AutoViz(
+        filename,
+        sep=sep,
+        depVar=depVar,
+        dfte=dfte,
+        header=header,
+        verbose=verbose,
+        lowess=lowess,
+        chart_format=chart_format,
+        max_rows_analyzed=max_rows_analyzed,
+        max_cols_analyzed=max_cols_analyzed,
+        save_plot_dir=save_plot_dir
+    )
+    print("save_plot_dir", save_plot_dir)
+    return {"message": "AutoViz run complete", "plots_dir": save_plot_dir}
+
+@Autoviz_router.post("/run_autoviz")
+async def autoviz_api(params: AutoVizParams):
+    result = run_autoviz(
+        filename=params.filename,
+        sep=params.sep,
+        depVar=params.depVar,
+        dfte=None,
+        header=params.header,
+        verbose=params.verbose,
+        lowess=params.lowess,
+        chart_format=params.chart_format,
+        max_rows_analyzed=params.max_rows_analyzed,
+        max_cols_analyzed=params.max_cols_analyzed,
+        save_plot_dir=None
+    )
+    return result

Routes/helpers/df_to_vis.py

@@ -0,0 +1,255 @@
+import re
+import uuid
+import pandas as pd
+import requests
+import tempfile
+import os
+from datetime import datetime
+from langchain_openai import ChatOpenAI
+from langchain.schema import SystemMessage, HumanMessage
+from nbformat import v4, write
+from nbclient import NotebookClient
+import sys
+from pathlib import Path
+import duckdb
+project_root = Path(__file__).resolve().parents[2]
+if str(project_root) not in sys.path:
+    sys.path.insert(0, str(project_root))
+from nbclient.exceptions import CellExecutionError
+from s3.read_files import read_csv_from_s3
+
+# Global flag
+IS_VIS = False
+
+
+def generate_plotly_visualization(
+    result_df: pd.DataFrame,
+    user_prompt: str,
+    is_vis: bool = IS_VIS,
+    upload_to_s3: bool = True,
+    s3_path: str = "vatsav/artifacts/"
+) -> dict:
+    """
+    Takes a pandas DataFrame and generates a Plotly visualization via LLM, 
+    saves it as a temporary HTML file, and optionally uploads to S3.
+    The temporary file is automatically cleaned up after upload.
+    
+    Args:
+        result_df (pd.DataFrame): Input DataFrame to visualize.
+        user_prompt (str): User prompt for visualization instructions.
+        is_vis (bool): Flag to enable/disable visualization generation. Default is global IS_VIS.
+        upload_to_s3 (bool): Flag to enable/disable S3 upload. Default is True.
+        s3_path (str): S3 path for upload. Default is "vatsav/artifacts/".
+        
+    Returns:
+        dict: Metadata dictionary with viz_id, timestamp, s3_url, and status.
+    """
+    
+    viz_id = str(uuid.uuid4())
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    
+    # If visualization is disabled, return early
+    if not is_vis:
+        metadata = {
+            "viz_id": viz_id,
+            "timestamp": timestamp,
+            "result": result_df.to_dict(),
+            "status": "success",
+            "message": "Visualization disabled (is_vis=False)",
+            "user_prompt": user_prompt
+        }
+        print("user_prompt:", user_prompt)
+        print(f"ℹ️ Visualization skipped (is_vis=False)")
+        return metadata
+    
+    try:
+        # 1. Prepare prompt for Plotly visualization assistant
+        plotly_agent_prompt = f""" 
+You are a cutting-edge Python visualization expert specializing in generating highly aesthetic and functional charts using the Plotly library. 
+
+Your task is to analyze the provided pandas DataFrame and generate a visualization that adheres to a smart, minimalist, and futuristic aesthetic.
+
+User request: "{user_prompt}"
+
+Follow the user request closely — especially regarding chart type or any explicit instructions.
+
+- Intelligent Data Analysis: Automatically analyze the DataFrame's columns to identify the optimal axis assignments (categorical/time-series for X, numeric/metric for Y). State the reason for the chosen assignment.
+- Smart Chart Selection: Determine and state the single most effective chart type (Bar, Line, Scatter, Histogram, Pie, etc.) that best represents the relationship.
+- Generate Plotly Python Code: Write concise, ready-to-run Python code using plotly.express or plotly.graph_objects.
+
+Aesthetic Requirements (Futuristic & Smart)
+- Theme: Apply a dark, high-tech, futuristic template with `template='plotly_dark'`.
+- Color Palette: Use high-contrast, digital-inspired colors [#4477A0, #FF9933, #B266CC, #33B2B2, #66CC66].
+- Labels & Title: Include a professional title and descriptive axes.
+- Annotation: For bar or pie charts, display metric values directly.
+
+Required Output Format
+- Suggested chart type: [Chart Type]
+- Analysis and Rationale: [Why chart type and columns were chosen]
+- Python Plotly code:
+        """
+        
+        # 2. Initialize ChatOpenAI
+        chat = ChatOpenAI(model="gpt-4o-mini", temperature=0.7)
+        
+        # 3. Prepare messages with system prompt and DataFrame as string
+        messages = [
+            SystemMessage(content=plotly_agent_prompt),
+            HumanMessage(content=result_df.to_string())
+        ]
+        
+        # 4. Get LLM response with Plotly code
+        response = chat.invoke(messages)  # Use invoke instead of __call__
+        response_content = response.content
+        
+        # 5. Extract Python Plotly code from response (between ```python ... ```)
+        code_match = re.search(r"```python\n(.*?)```", response_content, re.DOTALL)
+        plotly_code = code_match.group(1).strip() if code_match else response_content.strip()
+        
+        # 6. Execute the Plotly code in a notebook client and save the plot HTML
+        
+        # Create a new notebook object
+        nb = v4.new_notebook()
+        nb.cells.append(v4.new_code_cell(plotly_code))
+        client = NotebookClient(nb, timeout=60, kernel_name="python3")
+        
+        try:
+            client.execute()
+        except CellExecutionError as e:
+            raise RuntimeError(f"Error executing Plotly code: {e}")
+        
+        # Save notebook for debugging (optional)
+        with open("temp_notebook.ipynb", "w", encoding="utf-8") as f:
+            write(nb, f)
+        
+        # 7. Extract figure object by executing code in local scope
+        local_scope = {}
+        exec(plotly_code, {}, local_scope)
+        fig = local_scope.get("fig")
+        
+        if fig is None:
+            raise RuntimeError("No Plotly figure named 'fig' found in the generated code.")
+        
+        # 8. Create a temporary file for the HTML and write to it
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.html', prefix='plotly_viz_', delete=False, encoding='utf-8') as temp_file:
+            temp_path = temp_file.name
+        
+        # Write HTML to the temp file path (Plotly handles encoding internally)
+        fig.write_html(temp_path)
+        
+        print(f"✅ Plotly visualization saved to temporary file")
+        
+        try:
+            metadata = {
+                "viz_id": viz_id,
+                "timestamp": timestamp,
+                "status": "success"
+            }
+            
+            # 9. Upload to S3 if enabled
+            if upload_to_s3:
+                try:
+                    s3_url = upload_file_to_s3(temp_path, s3_path)
+                    metadata["s3_url"] = s3_url
+                    metadata["s3_status"] = "uploaded"
+                    print(f"✅ File uploaded to S3: {s3_url}")
+                except Exception as e:
+                    metadata["s3_status"] = "failed"
+                    metadata["s3_error"] = str(e)
+                    print(f"❌ S3 upload failed: {str(e)}")
+            
+            print(f"✅ Process completed (ID: {viz_id})")
+            
+            return metadata
+            
+        finally:
+            # 10. Clean up temporary file
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+                print(f"🗑️ Temporary file cleaned up")
+        
+    except Exception as e:
+        return {
+            "viz_id": viz_id,
+            "timestamp": timestamp,
+            "error": str(e),
+            "status": "failed"
+        }
+
+
+def upload_file_to_s3(file_path: str, s3_path: str = "vatsav/artifacts/") -> str:
+    """
+    Uploads a file to S3 using the HuggingFace Space API.
+    
+    Args:
+        file_path (str): Local path to the file to upload.
+        s3_path (str): S3 path where the file should be uploaded.
+        
+    Returns:
+        str: S3 URL of the uploaded file.
+        
+    Raises:
+        Exception: If upload fails.
+    """
+    api_url = f"https://srivatsavdamaraju-mvp-2-0-deploy-all-apis.hf.space/s3/upload/?path={s3_path}"
+    
+    # Determine content type based on file extension
+    if file_path.endswith('.html'):
+        content_type = 'text/html'
+    elif file_path.endswith('.json'):
+        content_type = 'application/json'
+    else:
+        content_type = 'application/octet-stream'
+    
+    # Open and upload the file
+    with open(file_path, 'rb') as f:
+        files = {
+            'file': (Path(file_path).name, f, content_type)
+        }
+        headers = {
+            'accept': 'application/json'
+        }
+        
+        response = requests.post(api_url, headers=headers, files=files)
+        
+        if response.status_code == 200:
+            response_data = response.json()
+            # Assuming the API returns the S3 URL in the response
+            s3_url = response_data.get('url') or response_data.get('file_url') or f"{s3_path}{Path(file_path).name}"
+            return s3_url
+        else:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+
+
+# # Example usage:
+# data = {
+#     'smoking_status': ['formerly smoked', 'smokes', 'never smoked', 'Unknown'],
+#     'stroke_percentage': [66.04, 51.22, 50.00, 39.17]
+# }
+
+# # Create the DataFrame
+# df = pd.DataFrame(data)
+
+# # Without visualization
+# # result = generate_plotly_visualization(df, user_prompt="", is_vis=False)
+# # print(result)
+
+# # With visualization and S3 upload (temp file auto-cleaned)
+# user_query = "show me a pie chart of smoking status vs stroke percentage"
+# result = generate_plotly_visualization(
+#     df, 
+#     user_prompt=user_query, 
+#     is_vis=True, 
+#     upload_to_s3=True,
+#     s3_path="vatsav/artifacts/tests/"
+# )
+# print(result)
+
+# With visualization but no S3 upload (temp file auto-cleaned)
+# result = generate_plotly_visualization(
+#     df, 
+#     user_prompt=user_query, 
+#     is_vis=True, 
+#     upload_to_s3=False
+# )
+# print(result)

Routes/helpers/duck_db_agent.py

@@ -0,0 +1,321 @@
+import asyncio
+import duckdb
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Annotated, Optional, List
+from functools import lru_cache
+
+import pandas as pd
+from pydantic import BaseModel, Field, ConfigDict
+from agents import Agent, Runner, function_tool
+import os
+import dotenv
+
+# =====================================
+# 🔹 CONFIGURATION
+# =====================================
+dotenv.load_dotenv()
+
+# Ensure project root is on sys.path
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# Validate environment
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY not found in environment variables")
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+# Local import after path setup
+from s3.read_files import read_csv_from_s3
+
+# Constants
+S3_DATASET_PATH = "vatsav/csv/mini_health-care_dataset.csv"
+ALLOWED_SQL_KEYWORDS = ("select", "with")
+MAX_RESULT_ROWS = 10000
+
+
+# =====================================
+# 🔹 PYDANTIC MODELS (Strict Schema Compatible)
+# =====================================
+class SQLResult(BaseModel):
+    """Result from SQL query execution."""
+    model_config = ConfigDict(extra='forbid')
+    
+    sql_query: str = Field(..., description="The executed SQL query")
+    result_summary: str = Field(..., description="Human-readable summary of results")
+    columns: List[str] = Field(..., description="Column names from result")
+    row_count: int = Field(..., description="Number of rows returned")
+    timestamp: str = Field(..., description="Execution timestamp")
+    execution_time_ms: float = Field(..., description="Query execution time in milliseconds")
+
+
+class PlotResult(BaseModel):
+    """Result from plot generation."""
+    model_config = ConfigDict(extra='forbid')
+    
+    viz_id: str = Field(..., description="Unique visualization identifier")
+    file_path: str = Field(..., description="Path to saved visualization file")
+    status: str = Field(..., description="Status of visualization generation")
+    chart_type: str = Field(..., description="Type of chart created")
+    timestamp: str = Field(..., description="Generation timestamp")
+
+
+# =====================================
+# 🔹 DATA LOADING WITH CACHING
+# =====================================
+@lru_cache(maxsize=1)
+def get_cached_dataset(s3_path: str) -> pd.DataFrame:
+    """Load dataset from S3 with caching."""
+    print(f"📥 Loading dataset from S3: {s3_path}")
+    df = read_csv_from_s3(s3_path)
+    print(f"✅ Dataset loaded: {len(df)} rows, {len(df.columns)} columns")
+    return df
+
+
+# Store the last query result globally for use by plotly_tool
+_last_query_result = None
+
+
+# =====================================
+# 🔹 SQL TOOL — performs data analytics
+# =====================================
+@function_tool
+def sql_tool(sql_query: Annotated[str, "SQL query to execute on healthcare dataset"]) -> SQLResult:
+    """
+    Execute SQL query on healthcare dataset (table: CSV1) and return results with metadata.
+    
+    Supported operations: SELECT, WITH (CTE)
+    Available table: CSV1 (healthcare dataset)
+    
+    Example queries:
+    - SELECT diagnosis, COUNT(*) as count FROM CSV1 GROUP BY diagnosis
+    - SELECT * FROM CSV1 WHERE age > 50 LIMIT 10
+    """
+    global _last_query_result
+    
+    start_time = datetime.now()
+    timestamp = start_time.strftime("%Y-%m-%d_%H-%M-%S")
+
+    try:
+        # Validate query
+        normalized_query = sql_query.strip().lower()
+        if not normalized_query.startswith(ALLOWED_SQL_KEYWORDS):
+            raise ValueError(
+                f"Only {', '.join(k.upper() for k in ALLOWED_SQL_KEYWORDS)} queries are allowed."
+            )
+
+        # Load dataset (cached)
+        df = get_cached_dataset(S3_DATASET_PATH)
+        
+        # Execute query using DuckDB
+        conn = duckdb.connect(":memory:")
+        conn.register("CSV1", df)
+        
+        result_df = conn.execute(sql_query).df()
+        conn.close()
+        
+        # Validate result size
+        if len(result_df) > MAX_RESULT_ROWS:
+            result_df = result_df.head(MAX_RESULT_ROWS)
+        
+        # Calculate execution time
+        execution_time = (datetime.now() - start_time).total_seconds() * 1000
+        
+        # Store result for potential visualization
+        _last_query_result = result_df.copy()
+        
+        # Create human-readable summary
+        summary_lines = [
+            f"Query returned {len(result_df)} rows with {len(result_df.columns)} columns: {', '.join(result_df.columns.tolist())}"
+        ]
+        
+        # Add preview of results
+        if not result_df.empty:
+            summary_lines.append("\nResults preview:")
+            # Format as table string
+            preview = result_df.head(10).to_string(index=False, max_rows=10)
+            summary_lines.append(preview)
+        
+        result_summary = "\n".join(summary_lines)
+        
+        print(f"✅ SQL executed: {len(result_df)} rows in {execution_time:.2f}ms")
+
+        return SQLResult(
+            sql_query=sql_query,
+            result_summary=result_summary,
+            columns=result_df.columns.tolist(),
+            row_count=len(result_df),
+            timestamp=timestamp,
+            execution_time_ms=round(execution_time, 2)
+        )
+
+    except ValueError as ve:
+        raise ValueError(f"SQL validation error: {str(ve)}")
+    except Exception as e:
+        raise RuntimeError(f"SQL execution error: {str(e)}")
+
+
+# =====================================
+# 🔹 PLOTLY TOOL — generates visualization
+# =====================================
+@function_tool
+def plotly_tool(
+    chart_type: Annotated[str, "Type of chart: bar, line, scatter, or pie"] = "bar",
+    x_column: Annotated[Optional[str], "Column name for x-axis"] = None,
+    y_column: Annotated[Optional[str], "Column name for y-axis"] = None,
+    title: Annotated[Optional[str], "Custom chart title"] = None
+) -> PlotResult:
+    """
+    Generate interactive Plotly visualization from the last SQL query result.
+    Must be called after sql_tool.
+    """
+    global _last_query_result
+    
+    import plotly.express as px
+    import plotly.graph_objects as go
+    from uuid import uuid4
+    
+    if _last_query_result is None:
+        raise ValueError("No SQL query result available. Please run sql_tool first.")
+    
+    viz_id = str(uuid4())[:8]
+    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    output_dir = PROJECT_ROOT / "outputs" / "visualizations"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    file_path = output_dir / f"viz_{viz_id}_{timestamp}.html"
+    
+    try:
+        df = _last_query_result.copy()
+        
+        if df.empty:
+            raise ValueError("Cannot create visualization from empty result set")
+        
+        # Auto-select x and y columns if not provided
+        if x_column is None:
+            x_column = df.columns[0]
+        if y_column is None:
+            y_column = df.columns[1] if len(df.columns) > 1 else df.columns[0]
+        
+        # Validate columns exist
+        if x_column not in df.columns:
+            raise ValueError(f"Column '{x_column}' not found. Available: {list(df.columns)}")
+        if y_column not in df.columns:
+            raise ValueError(f"Column '{y_column}' not found. Available: {list(df.columns)}")
+        
+        # Generate chart title
+        if title is None:
+            title = f"{y_column} by {x_column}"
+        
+        # Generate appropriate chart
+        chart_mapping = {
+            "bar": px.bar,
+            "line": px.line,
+            "scatter": px.scatter,
+            "pie": px.pie
+        }
+        
+        plot_func = chart_mapping.get(chart_type.lower(), px.bar)
+        
+        if chart_type.lower() == "pie":
+            fig = plot_func(df, names=x_column, values=y_column, title=title)
+        else:
+            fig = plot_func(df, x=x_column, y=y_column, title=title)
+        
+        # Enhanced styling
+        fig.update_layout(
+            template="plotly_white",
+            height=600,
+            showlegend=True,
+            font=dict(size=12),
+            title_font_size=16
+        )
+        
+        fig.write_html(str(file_path))
+        print(f"📊 Visualization created: {chart_type} chart saved to {file_path.name}")
+        
+        return PlotResult(
+            viz_id=viz_id,
+            file_path=str(file_path),
+            status="success",
+            chart_type=chart_type,
+            timestamp=timestamp
+        )
+        
+    except Exception as e:
+        raise RuntimeError(f"Plotly error: {str(e)}")
+
+
+# =====================================
+# 🔹 AGENT DEFINITION
+# =====================================
+agent = Agent(
+    name="Healthcare Data Analyst",
+    instructions=(
+        "You are a healthcare data analyst. Analyze queries and generate SQL + visualizations.\n\n"
+        "Available dataset: CSV1 (healthcare patient data)\n\n"
+        "Workflow:\n"
+        "1. Generate SQL query using sql_tool to analyze CSV1\n"
+        "2. Review the result_summary to understand the data\n"
+        "3. If visualization is needed, use plotly_tool (it uses the last SQL result)\n"
+        "4. Provide clear insights with context\n\n"
+        "SQL Best Practices:\n"
+        "- Use descriptive aliases\n"
+        "- Add ORDER BY for readability\n"
+        "- Use LIMIT for large datasets\n\n"
+        "Visualization Guidelines:\n"
+        "- bar: counts, comparisons\n"
+        "- line: trends, time series\n"
+        "- pie: proportions (5-7 categories max)\n"
+        "- scatter: correlations\n\n"
+        "Always interpret results for non-technical users."
+    ),
+    tools=[sql_tool],
+)
+
+
+# =====================================
+# 🔹 MAIN RUNNER
+# =====================================
+async def run_analysis(user_query: str, verbose: bool = True) -> str:
+    """Run the agent with the given query and return results."""
+    if verbose:
+        print(f"\n{'='*70}")
+        print(f"📋 Query: {user_query}")
+        print(f"{'='*70}\n")
+    
+    try:
+        result = await Runner.run(agent, input=user_query)
+        return result.final_output
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        raise
+
+
+async def main():
+    """Main entry point."""
+    user_query = (
+        "Show how many patients exist in each diagnosis category, "
+        "filtered for married individuals. Create a bar chart to visualize this."
+    )
+    
+    try:
+        output = await run_analysis(user_query)
+        
+        # Clean output formatting
+        print("\n" + "="*70)
+        print("📊 ANALYSIS COMPLETE")
+        print("="*70)
+        print(output)
+        print("="*70 + "\n")
+        
+    except Exception as e:
+        print(f"\n❌ Analysis failed: {e}\n")
+        raise
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

Routes/helpers/main_agent_best_as_of_now.py

@@ -0,0 +1,700 @@
+import duckdb
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Annotated, Optional, List
+import re
+import json
+import pandas as pd
+from pydantic import BaseModel, Field, ConfigDict
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import JSONResponse
+import os
+import dotenv
+import tempfile
+from langchain_openai import ChatOpenAI
+from langchain.schema import SystemMessage, HumanMessage
+import requests
+import asyncio
+import sqlparse
+
+# Project root setup
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# Assuming these are custom modules in your project
+from agents import Agent, Runner, function_tool
+from s3.read_files import read_csv_from_s3
+
+# =====================================
+# 🔹 CONFIGURATION
+# =====================================
+dotenv.load_dotenv()
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY not found in environment variables")
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+# Constants
+ALLOWED_SQL_KEYWORDS = ("select", "with")
+MAX_RESULT_ROWS = 10000
+
+# Global storage for datasets and context
+_registered_datasets = {}
+_dataset_context = {}
+last_sql_query: str = ""
+chartjs_path = []  # Global variable to store Chart.js JSON config path or S3 URL
+
+def is_safe_query(sql_query: str) -> bool:
+    parsed = sqlparse.parse(sql_query)[0]
+    forbidden_keywords = {"DROP", "DELETE", "UPDATE", "INSERT"}
+    tokens = [token.value.upper() for token in parsed.tokens if token.is_keyword]
+    return not any(keyword in forbidden_keywords for keyword in tokens)
+
+# =====================================
+# 🔹 S3 UPLOAD FUNCTION
+# =====================================
+
+# =====================================
+# 🔹 UTILITY FUNCTIONS
+# =====================================
+def save_convo_id_to_pgdb(convo_id: str):
+    """Save conversation ID to PostgreSQL database (stub function)."""
+    print(f"Saving conversation ID to PostgreSQL DB: {convo_id}")
+    pass
+
+# =====================================
+# 🔹 PYDANTIC MODELS
+# =====================================
+class AnalysisRequest(BaseModel):
+    """Request model for data analysis."""
+    file_paths: List[str] = Field(..., description="List of S3 file paths to analyze")
+    query: str = Field(..., description="Natural language query for analysis")
+    
+    model_config = ConfigDict(extra='forbid')
+
+class SQLResult(BaseModel):
+    """Result from SQL query execution."""
+    model_config = ConfigDict(extra='forbid')
+    
+    sql_query: str = Field(..., description="The executed SQL query")
+    result_summary: str = Field(..., description="Human-readable summary of results")
+    columns: List[str] = Field(..., description="Column names from result")
+    row_count: int = Field(..., description="Number of rows returned")
+    execution_time_ms: float = Field(..., description="Query execution time in milliseconds")
+
+class AnalysisResponse(BaseModel):
+    """Response from analysis endpoint."""
+    model_config = ConfigDict(extra='forbid')
+    
+    query: str = Field(..., description="Original user query")
+    datasets_loaded: List[str] = Field(..., description="List of loaded datasets")
+    analysis_result: str = Field(..., description="Analysis insights")
+    sql_executed: str = Field(..., description="SQL query that was executed")
+    execution_time_ms: float = Field(..., description="Total execution time")
+
+# =====================================
+# 🔹 DATASET MANAGEMENT & CONTEXT ENGINEERING 
+# =====================================
+def generate_dataset_context(df: pd.DataFrame, table_name: str) -> str:
+    """Generate comprehensive context for a dataset."""
+    context_parts = []
+    
+    context_parts.append(f"Table: {table_name}")
+    context_parts.append(f"Rows: {len(df)}, Columns: {len(df.columns)}")
+    context_parts.append(f"Column Names: {', '.join(df.columns.tolist())}")
+    
+    context_parts.append("\nData Types:")
+    for col, dtype in df.dtypes.items():
+        context_parts.append(f"  - {col}: {dtype}")
+    
+    context_parts.append("\nSample Data (first 3 rows):")
+    sample_str = df.head(3).to_string(index=False, max_cols=10)
+    context_parts.append(sample_str)
+    
+    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+    if numeric_cols:
+        context_parts.append("\nNumeric Column Statistics:")
+        desc = df[numeric_cols].describe().transpose()
+        for col in numeric_cols[:5]:
+            if col in desc.index:
+                context_parts.append(f"  - {col}: mean={desc.loc[col, 'mean']:.2f}, "
+                                   f"min={desc.loc[col, 'min']:.2f}, max={desc.loc[col, 'max']:.2f}")
+    
+    categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+    if categorical_cols:
+        context_parts.append("\nCategorical Columns:")
+        for col in categorical_cols[:5]:
+            unique_count = df[col].nunique()
+            context_parts.append(f"  - {col}: {unique_count} unique values")
+            if unique_count <= 5:
+                values = df[col].value_counts().head(5).to_dict()
+                context_parts.append(f"    Values: {values}")
+    
+    return "\n".join(context_parts)
+
+def load_and_register_datasets(file_paths: List[str]) -> str:
+    """Load datasets from S3 and register them in DuckDB format."""
+    global _registered_datasets, _dataset_context
+    
+    _registered_datasets.clear()
+    _dataset_context.clear()
+    
+    context_parts = ["Available datasets for SQL queries:\n"]
+    
+    for idx, file_path in enumerate(file_paths, start=1):
+        table_name = f"CSV{idx}"
+        
+        try:
+            print(f"📥 Loading {file_path} as {table_name}")
+            df = read_csv_from_s3(file_path)
+            
+            _registered_datasets[table_name] = df
+            
+            dataset_context = generate_dataset_context(df, table_name)
+            _dataset_context[table_name] = dataset_context
+            context_parts.append(f"\n{dataset_context}\n")
+            context_parts.append("="*70)
+            
+            print(f"✅ Loaded {table_name}: {len(df)} rows, {len(df.columns)} columns")
+            
+        except Exception as e:
+            print(f"❌ Error loading {file_path}: {str(e)}")
+            raise HTTPException(status_code=400, detail=f"Failed to load {file_path}: {str(e)}")
+    
+    full_context = "\n".join(context_parts)
+    return full_context
+
+# =====================================
+# 🔹 CHART.JS VISUALIZATION FUNCTION
+# =====================================
+def generate_chartjs_visualization(result_df: pd.DataFrame, user_prompt: str) -> dict:
+    """
+    Generate Chart.js JSON config from DataFrame using LLM.
+    
+    Args:
+        result_df: pandas DataFrame to visualize
+        user_prompt: User's visualization request
+        
+    Returns:
+        dict: Chart.js configuration JSON
+    """
+    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)
+    
+    prompt = f"""You are an expert data visualization analyst. Analyze the DataFrame and user query to generate the BEST Chart.js visualization.
+
+User Query: "{user_prompt}"
+
+Your Task:
+1. ANALYZE the DataFrame structure:
+   - Identify column types (categorical, numerical, temporal)
+   - Detect relationships between columns
+   - Understand data distribution and patterns
+
+2. DETERMINE the optimal visualization:
+   - For comparisons → bar chart
+   - For trends over time → line chart
+   - For proportions/parts of whole → pie/doughnut
+   - For distributions → scatter/bubble
+   - For multivariate comparisons → radar/polarArea
+   - Match chart type to data characteristics AND user intent
+
+3. GENERATE Chart.js JSON with:
+   - Correct axis mappings (categorical → labels, numerical → data)
+   - Meaningful chart title derived from the data context
+   - Appropriate colors for distinction
+   - Professional styling options
+
+Available chart types: bar, line, pie, doughnut, polarArea, radar, scatter, bubble
+
+Color palette: #FF6384, #36A2EB, #FFCE56, #4BC0C0, #9966FF, #FF9F40
+
+**CRITICAL**: Output ONLY valid JSON with all property names enclosed in double quotes. Do NOT include any text, comments, or explanations outside the JSON. Ensure the JSON is complete and properly formatted. All property names MUST be enclosed in double quotes.
+
+Example of valid output:
+{{
+  "type": "bar",
+  "data": {{
+    "labels": ["A", "B", "C"],
+    "datasets": [{{
+      "label": "Sample Data",
+      "data": [10, 20, 30],
+      "backgroundColor": ["#FF6384", "#36A2EB", "#FFCE56"],
+      "borderColor": ["#FF6384", "#36A2EB", "#FFCE56"],
+      "borderWidth": 2
+    }}]
+  }},
+  "options": {{
+    "responsive": true,
+    "maintainAspectRatio": false,
+    "plugins": {{
+      "title": {{
+        "display": true,
+        "text": "Sample Bar Chart",
+        "font": {{ "size": 16 }}
+      }},
+      "legend": {{
+        "display": true,
+        "position": "top"
+      }}
+    }},
+    "scales": {{
+      "y": {{
+        "beginAtZero": true,
+        "title": {{ "display": true, "text": "Values" }}
+      }},
+      "x": {{
+        "title": {{ "display": true, "text": "Categories" }}
+      }}
+    }}
+  }}
+}}
+
+Output ONLY the JSON content:
+{{
+  "type": "best_chart_type",
+  "data": {{
+    "labels": ["derived", "from", "dataframe"],
+    "datasets": [{{
+      "label": "Meaningful Label",
+      "data": [actual, values, from, df],
+      "backgroundColor": ["#FF6384", "#36A2EB", "#FFCE56"],
+      "borderColor": ["#FF6384", "#36A2EB", "#FFCE56"],
+      "borderWidth": 2
+    }}]
+  }},
+  "options": {{
+    "responsive": true,
+    "maintainAspectRatio": false,
+    "plugins": {{
+      "title": {{
+        "display": true,
+        "text": "Descriptive Title Based on Data",
+        "font": {{ "size": 16 }}
+      }},
+      "legend": {{
+        "display": true,
+        "position": "top"
+      }}
+    }},
+    "scales": {{
+      "y": {{
+        "beginAtZero": true,
+        "title": {{ "display": true, "text": "Y Axis Label" }}
+      }},
+      "x": {{
+        "title": {{ "display": true, "text": "X Axis Label" }}
+      }}
+    }}
+  }}
+}}
+"""
+    
+    messages = [
+        SystemMessage(content=prompt),
+        HumanMessage(content=result_df.to_string())
+    ]
+    
+    response = llm.invoke(messages)
+    print(f"Raw LLM Response:\n{response.content}\n")
+    
+    json_match = re.search(r"\{.*\}", response.content, re.DOTALL)
+    if not json_match:
+        raise ValueError(f"No valid JSON block found in LLM response:\n{response.content}")
+    
+    json_string = json_match.group(0).strip()
+    print(f"Extracted JSON String:\n{json_string}\n")
+    
+    try:
+        config = json.loads(json_string)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON format: {str(e)}\nJSON String:\n{json_string}")
+    
+    print(f"Generated Chart.js Config: {json.dumps(config, indent=2)}")
+    
+    if "type" not in config or "data" not in config:
+        raise ValueError("Invalid Chart.js config")
+    
+    return config
+
+# =====================================
+# 🔹 VECTOR AGENT TOOL
+# =====================================
+def query_vector_agent_calling(user_query: str, collection_name: str) -> str:
+    """
+    Call the vector agent API to get relevant context for the given user query and collection name.
+    """
+    base_url = "https://srivatsavdamaraju-mvp-2-0-deploy-all-apis.hf.space/qdrant/search"
+    url = f"{base_url}?collection_name={collection_name}&mode=dense"
+    
+    headers = {
+        "accept": "application/json",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "query": user_query,
+        "top_k": 2
+    }
+
+    try:
+        response = requests.post(url, headers=headers, json=payload)
+        response.raise_for_status()
+
+        data = response.json()
+        results = data.get("results") or data.get("result") or data
+        if not results:
+            return "No relevant context found."
+
+        contexts = []
+        for item in results:
+            text = item.get("text") or item.get("payload", {}).get("text")
+            if text:
+                contexts.append(text)
+        
+        return "\n\n".join(contexts) if contexts else str(data)
+
+    except requests.RequestException as e:
+        print(f"Error calling vector agent API: {e}")
+        return "Error retrieving context."
+
+@function_tool
+async def vector_agent_tool(
+    user_query: Annotated[str, "The user's natural language question"],
+    collection_name: Annotated[str, "The name of the collection to search in"]
+) -> str:
+    """
+    Tool to call the vector agent API and retrieve relevant context.
+    
+    Args:
+        user_query: The user's natural language question.
+        collection_name: The name of the collection to search in.
+    
+    Returns:
+        A string containing the retrieved context.
+    """
+    return query_vector_agent_calling(user_query, collection_name)
+
+# =====================================
+# 🔹 SQL TOOL
+# =====================================
+@function_tool
+async def sql_tool(
+    sql_query: Annotated[str, "SQL query to execute on registered datasets"],
+    user_query: Annotated[str, "Original user question for visualization context"] = ""
+) -> SQLResult:
+    """
+    Execute SQL query on registered datasets and return results.
+    
+    Available tables: CSV1, CSV2, CSV3, etc. (based on loaded files)
+    Supported operations: SELECT, WITH (CTE)
+    
+    Args:
+        sql_query: The SQL query to execute
+        user_query: The original user question (REQUIRED for visualization context)
+    
+    Example queries:
+    - SELECT * FROM CSV1 WHERE age > 50 LIMIT 10
+    - SELECT a.name, b.value FROM CSV1 a JOIN CSV2 b ON a.id = b.id
+    
+    Example usage:
+    - sql_tool(
+        sql_query="SELECT * FROM CSV1",
+        user_query="What is the average age?"
+    )
+    """
+    global _registered_datasets, last_sql_query
+
+    last_sql_query = sql_query
+
+    start_time = datetime.now()
+    print(f"\n{'='*70}")
+    print(f"🔍 Executing SQL Query:\n{sql_query}\n")
+    print(f"📝 User Query Context: {user_query}\n")
+    print(f"{'='*70}\n")
+
+    try:
+        normalized_query = sql_query.strip().lower()
+        if not normalized_query.startswith(ALLOWED_SQL_KEYWORDS):
+            raise ValueError(
+                f"Only {', '.join(k.upper() for k in ALLOWED_SQL_KEYWORDS)} queries are allowed."
+            )
+        
+        if not _registered_datasets:
+            raise ValueError("No datasets loaded. Please provide file_paths first.")
+
+        conn = duckdb.connect(":memory:")
+        
+        for table_name, df in _registered_datasets.items():
+            conn.register(table_name, df)
+        
+        result_df = conn.execute(sql_query).df()
+        conn.close()
+        
+        if len(result_df) > MAX_RESULT_ROWS:
+            result_df = result_df.head(MAX_RESULT_ROWS)
+        
+        execution_time = (datetime.now() - start_time).total_seconds() * 1000
+        
+        summary_lines = [
+            f"Query returned {len(result_df)} rows with {len(result_df.columns)} columns: {', '.join(result_df.columns.tolist())}"
+        ]
+        
+        if not result_df.empty:
+            summary_lines.append("\nResults preview (first 10 rows):")
+            preview = result_df.head(10).to_string(index=False, max_rows=10)
+            summary_lines.append(preview)
+        
+        result_summary = "\n".join(summary_lines)
+        
+        print(f"✅ SQL executed: {len(result_df)} rows in {execution_time:.2f}ms\n")
+
+        return SQLResult(
+            sql_query=sql_query,
+            result_summary=result_summary,
+            columns=result_df.columns.tolist(),
+            row_count=len(result_df),
+            execution_time_ms=round(execution_time, 2),
+        )
+
+    except ValueError as ve:
+        raise ValueError(f"SQL validation error: {str(ve)}")
+    except Exception as e:
+        raise RuntimeError(f"SQL execution error: {str(e)}")
+
+# =====================================
+# 🔹 VISUALIZATION TASK
+# =====================================
+async def generate_visualization_task(query: str) -> Optional[dict]:
+    """Async task to generate Chart.js visualization using last_sql_query and upload to S3."""
+    global _registered_datasets, last_sql_query, chartjs_path
+    
+    if not last_sql_query or not _registered_datasets:
+        print("⚠️ No SQL query executed or no datasets loaded, skipping visualization.")
+        return None
+
+    print(f"🎨 Generating Chart.js visualization...")
+    try:
+        conn = duckdb.connect(":memory:")
+        for table_name, df in _registered_datasets.items():
+            conn.register(table_name, df)
+        result_df = conn.execute(last_sql_query).df()
+        conn.close()
+
+        visualization_prompt = query.strip() if query and query.strip() else "Visualize the query results"
+        print(f"🎯 User prompt for visualization: '{visualization_prompt}'")
+        
+        chartjs_config = generate_chartjs_visualization(
+            result_df=result_df,
+            user_prompt=visualization_prompt
+        )
+        print(f"✅ Chart.js visualization generated.")
+        print(f"Chart.js Config: {json.dumps(chartjs_config, indent=2)}")
+        
+        # Save Chart.js config to temporary file and upload to S3
+        chartjs_path.clear()
+        chartjs_path.append(json.dumps(chartjs_config, indent=2))  # Store locally in the global list
+
+        return chartjs_config
+        
+    except Exception as viz_error:
+        print(f"⚠️ Visualization generation failed: {str(viz_error)}")
+        print(f"📊 Continuing with results without visualization...")
+        return None
+
+# =====================================
+# 🔹 MAIN ANALYSIS FUNCTION
+# =====================================
+async def main(query: str, file_paths: List[str] = None) -> dict:
+    """Main function to run data analysis with proper agent initialization."""
+    
+    if file_paths is None:
+        file_paths = []
+    
+    print(f"\n{'='*70}")
+    print(f"📋 Analysis Request")
+    print(f"{'='*70}")
+    print(f"Files: {', '.join(file_paths)}")
+    print(f"Query: {query}")
+    print(f"{'='*70}\n")
+    
+    dataset_context = load_and_register_datasets(file_paths)
+    
+    print(f"\n{'='*70}")
+    print("📊 Dataset Context Summary")
+    print(f"{'='*70}")
+    print(dataset_context[:500] + "..." if len(dataset_context) > 500 else dataset_context)
+    print(f"{'='*70}\n")
+
+    general_purpose_agent = Agent(
+        name="general_purpose_agent",
+        instructions=(
+            "You are a friendly AI assistant with high expertise in data analysis, SQL, and visualization. "
+            "Only answer if the query is not related to data analysis."
+        ),
+        model="gpt-4o-mini",
+        tools=[]
+    )
+
+    DataAnalyst_agent = Agent(
+        name="DataAnalyst_agent",
+        instructions=(
+            f"""You are a professional data analyst with expertise in SQL and data analysis.
+
+{dataset_context}
+
+Your workflow:
+1. Analyze the user's question carefully
+2. Review the available datasets and their schemas above
+3. **Generate appropriate SQL query using sql_tool - ALWAYS pass the user_query parameter**
+   
+   CRITICAL - Tool Usage:
+   - Call sql_tool with BOTH parameters:
+     * sql_query: Your SQL SELECT statement
+     * user_query: The EXACT original user question (this creates the visualization)
+   
+   Example correct usage:
+   sql_tool(
+       sql_query="SELECT age, COUNT(*) as count FROM CSV1 GROUP BY age ORDER BY age",
+       user_query="Show me the age distribution of patients"
+   )
+   
+   - Use table names: CSV1, CSV2, CSV3, etc.
+   - You can JOIN tables if needed
+   - Use descriptive column aliases
+   - Add ORDER BY for readability
+   - Use LIMIT for large result sets
+   
+4. Interpret the results and provide clear, detailed insights with specific numbers and percentages
+
+SQL Best Practices:
+- Always check which columns exist in which tables
+- Use table aliases for clarity (e.g., CSV1 AS c1)
+- Filter data appropriately based on the question
+- Aggregate data when asked for counts, sums, averages
+- Sort results logically
+
+Important:
+- Only use columns that exist in the datasets above
+- Reference the correct table names (CSV1, CSV2, etc.)
+- **MANDATORY: Always pass both sql_query AND user_query to sql_tool**
+- The user_query parameter is used to generate intelligent visualizations
+- Always provide complete analysis with specific findings
+- Include tables, numbers, and percentages in your response
+- Provide insights in plain language for non-technical users
+            """
+        ),
+        model="gpt-4o-mini",
+        tools=[sql_tool],
+    )
+
+    orchestrator_agent = Agent(
+        name="orchestrator_agent",
+        instructions=(
+            "You receive the user's query and route it to the DataAnalyst agent for execution. "
+            "Always use the analyze_dataset tool to process data analysis queries. "
+            "Pass the complete user query to the DataAnalyst agent."
+        ),
+        tools=[
+            DataAnalyst_agent.as_tool(
+                tool_name="analyze_dataset",
+                tool_description="Analyze dataset, generate SQL, and visualize results."
+            ),
+            general_purpose_agent.as_tool(
+                tool_name="general_purpose_tool",
+                tool_description="Act like a human friendly AI assistant for non-data analysis queries."
+            )
+        ],
+        model="gpt-4o-mini",
+    )
+    
+    print(f"\n{'='*70}")
+    print("🚀 Running Orchestrator Agent")
+    print(f"{'='*70}\n")
+    
+    orchestrator_result = await Runner.run(orchestrator_agent, query)
+
+    print(f"\n{'='*70}")
+    print("📝 Orchestrator Output")
+    print(f"{'='*70}")
+    
+    analysis_text = ""
+    for item in orchestrator_result.new_items:
+        if hasattr(item, 'output') and item.output:
+            analysis_text = item.output
+            print(f"Found analysis output: {len(analysis_text)} characters")
+        elif hasattr(item, 'raw_item') and hasattr(item.raw_item, 'content'):
+            for content_item in item.raw_item.content:
+                if hasattr(content_item, 'text'):
+                    analysis_text = content_item.text
+                    print(f"Found message output: {len(analysis_text)} characters")
+    
+    print(f"{'='*70}\n")
+
+    synthesizer_agent = Agent(
+        name="synthesizer_agent",
+        instructions=(
+            """You are a final report writer. Your job is to present the complete data analysis results.
+
+CRITICAL RULES:
+1. Present the COMPLETE analysis from the DataAnalyst agent
+2. Include ALL tables, numbers, statistics, and findings
+3. Maintain all the detailed insights and percentages
+4. Use clear formatting with headers, tables, and bullet points
+5. DO NOT summarize or truncate the analysis
+6. DO NOT add generic closing statements like "feel free to ask"
+7. Make the report professional and easy to read
+
+Your output should be the full, detailed analysis report with all findings intact.
+            """
+        ),
+        model="gpt-4o-mini",
+    )
+    
+    print(f"\n{'='*70}")
+    print("🎯 Running Synthesizer and Visualization in Parallel")
+    print(f"{'='*70}\n")
+    
+    synthesizer_task = asyncio.create_task(
+        Runner.run(
+            synthesizer_agent,
+            f"Present this complete data analysis:\n\n{analysis_text}" if analysis_text else orchestrator_result.to_input_list()
+        )
+    )
+    visualization_task = asyncio.create_task(generate_visualization_task(query))
+    
+    synthesizer_result, visualization_result = await asyncio.gather(synthesizer_task, visualization_task, return_exceptions=True)
+    
+    if isinstance(synthesizer_result, Exception):
+        print(f"⚠️ Synthesizer task failed: {str(synthesizer_result)}")
+        synthesizer_result = type('Result', (), {'final_output': 'Synthesizer task failed'})()
+    
+    if isinstance(visualization_result, Exception):
+        print(f"⚠️ Visualization task failed: {str(visualization_result)}")
+        visualization_result = None
+
+    print(f"\n{'='*70}")
+    print("✅ FINAL ANSWER")
+    print(f"{'='*70}")
+    print(synthesizer_result.final_output)
+    print(f"{'='*70}\n")
+    
+    print(f"chartjs_config_path: {chartjs_path}")
+    print(f"{'='*70}\n")
+    
+    final_api_output = {
+        "convoId": "1234",
+        "type": "assistant_message",
+        "hidden_code": True if last_sql_query else False,
+        "hidden_text": last_sql_query,
+        "ai_message": synthesizer_result.final_output,
+        "artifacts": chartjs_path
+    }
+    # print(f"Final JSON Output: {json.dumps(final_api_output, indent=2)}")
+    final_api_output_json = json.dumps(final_api_output, indent=2)
+    return final_api_output_json
+
+# =====================================
+# 🔹 STANDALONE TESTING

Routes/helpers/main_agent_best_as_of_now.py-update.txt

@@ -0,0 +1,1110 @@
+import duckdb
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Annotated, Optional, List
+import re
+import json
+import pandas as pd
+from pydantic import BaseModel, Field, ConfigDict
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import JSONResponse
+import os
+import dotenv
+import tempfile
+from langchain_openai import ChatOpenAI
+from langchain.schema import SystemMessage, HumanMessage
+import requests
+import asyncio
+# Project root setup
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+# Assuming these are custom modules in your project
+from agents import Agent, Runner, function_tool
+from s3.read_files import read_csv_from_s3
+
+# =====================================
+# 🔹 CONFIGURATION
+# =====================================
+dotenv.load_dotenv()
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+if not OPENAI_API_KEY:
+    raise ValueError("OPENAI_API_KEY not found in environment variables")
+os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
+
+# Constants
+ALLOWED_SQL_KEYWORDS = ("select", "with")
+MAX_RESULT_ROWS = 10000
+
+# Global storage for datasets and context
+_registered_datasets = {}
+_dataset_context = {}
+last_sql_query: str = ""
+vega_lite_path = []  # Global variable to store Vega-Lite JSON config path
+
+import sqlparse
+
+def is_safe_query(sql_query: str) -> bool:
+    parsed = sqlparse.parse(sql_query)[0]
+    forbidden_keywords = {"DROP", "DELETE", "UPDATE", "INSERT"}
+    tokens = [token.value.upper() for token in parsed.tokens if token.is_keyword]
+    return not any(keyword in forbidden_keywords for keyword in tokens)
+
+
+# =====================================
+# 🔹 S3 UPLOAD FUNCTION
+# =====================================
+def upload_file_to_s3(file_path: str, s3_path: str = "vatsav/artifacts/") -> str:
+    """
+    Uploads a file to S3 using the HuggingFace Space API.
+    
+    Args:
+        file_path (str): Local path to the file to upload.
+        s3_path (str): S3 path where the file should be uploaded.
+        
+    Returns:
+        str: S3 URL of the uploaded file.
+        
+    Raises:
+        Exception: If upload fails.
+    """
+    api_url = f"https://srivatsavdamaraju-mvp-2-0-deploy-all-apis.hf.space/s3/upload/?path={s3_path}"
+    
+    # Determine content type based on file extension
+    if file_path.endswith('.html'):
+        content_type = 'text/html'
+    elif file_path.endswith('.json'):
+        content_type = 'application/json'
+    else:
+        content_type = 'application/octet-stream'
+    
+    # Open and upload the file
+    with open(file_path, 'rb') as f:
+        files = {
+            'file': (Path(file_path).name, f, content_type)
+        }
+        headers = {
+            'accept': 'application/json'
+        }
+        
+        response = requests.post(api_url, headers=headers, files=files)
+        
+        if response.status_code == 200:
+            response_data = response.json()
+            s3_url = response_data.get('url') or response_data.get('file_url') or f"{s3_path}{Path(file_path).name}"
+            return s3_url
+        else:
+            raise Exception(f"Upload failed with status {response.status_code}: {response.text}")
+
+# ... [Previous code for dataset management, pydantic models, vector agent tool, SQL tool, and generate_chartjs_visualization remains unchanged]
+
+# =====================================
+# 🔹 UTILITY FUNCTIONS
+# =====================================
+def save_convo_id_to_pgdb(convo_id: str):
+    """Save conversation ID to PostgreSQL database (stub function)."""
+    print(f"Saving conversation ID to PostgreSQL DB: {convo_id}")
+    pass
+
+# =====================================
+# 🔹 PYDANTIC MODELS
+# =====================================
+class AnalysisRequest(BaseModel):
+    """Request model for data analysis."""
+    file_paths: List[str] = Field(..., description="List of S3 file paths to analyze")
+    query: str = Field(..., description="Natural language query for analysis")
+    
+    model_config = ConfigDict(extra='forbid')
+
+class SQLResult(BaseModel):
+    """Result from SQL query execution."""
+    model_config = ConfigDict(extra='forbid')
+    
+    sql_query: str = Field(..., description="The executed SQL query")
+    result_summary: str = Field(..., description="Human-readable summary of results")
+    columns: List[str] = Field(..., description="Column names from result")
+    row_count: int = Field(..., description="Number of rows returned")
+    execution_time_ms: float = Field(..., description="Query execution time in milliseconds")
+
+class AnalysisResponse(BaseModel):
+    """Response from analysis endpoint."""
+    model_config = ConfigDict(extra='forbid')
+    
+    query: str = Field(..., description="Original user query")
+    datasets_loaded: List[str] = Field(..., description="List of loaded datasets")
+    analysis_result: str = Field(..., description="Analysis insights")
+    sql_executed: str = Field(..., description="SQL query that was executed")
+    execution_time_ms: float = Field(..., description="Total execution time")
+
+# =====================================
+# 🔹 DATASET MANAGEMENT & CONTEXT ENGINEERING 
+# =====================================
+def generate_dataset_context(df: pd.DataFrame, table_name: str) -> str:
+    """Generate comprehensive context for a dataset."""
+    context_parts = []
+    
+    context_parts.append(f"Table: {table_name}")
+    context_parts.append(f"Rows: {len(df)}, Columns: {len(df.columns)}")
+    context_parts.append(f"Column Names: {', '.join(df.columns.tolist())}")
+    
+    context_parts.append("\nData Types:")
+    for col, dtype in df.dtypes.items():
+        context_parts.append(f"  - {col}: {dtype}")
+    
+    context_parts.append("\nSample Data (first 3 rows):")
+    sample_str = df.head(3).to_string(index=False, max_cols=10)
+    context_parts.append(sample_str)
+    
+    numeric_cols = df.select_dtypes(include=['number']).columns.tolist()
+    if numeric_cols:
+        context_parts.append("\nNumeric Column Statistics:")
+        desc = df[numeric_cols].describe().transpose()
+        for col in numeric_cols[:5]:
+            if col in desc.index:
+                context_parts.append(f"  - {col}: mean={desc.loc[col, 'mean']:.2f}, "
+                                   f"min={desc.loc[col, 'min']:.2f}, max={desc.loc[col, 'max']:.2f}")
+    
+    categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()
+    if categorical_cols:
+        context_parts.append("\nCategorical Columns:")
+        for col in categorical_cols[:5]:
+            unique_count = df[col].nunique()
+            context_parts.append(f"  - {col}: {unique_count} unique values")
+            if unique_count <= 5:
+                values = df[col].value_counts().head(5).to_dict()
+                context_parts.append(f"    Values: {values}")
+    
+    return "\n".join(context_parts)
+
+def load_and_register_datasets(file_paths: List[str]) -> str:
+    """Load datasets from S3 and register them in DuckDB format."""
+    global _registered_datasets, _dataset_context
+    
+    _registered_datasets.clear()
+    _dataset_context.clear()
+    
+    context_parts = ["Available datasets for SQL queries:\n"]
+    
+    for idx, file_path in enumerate(file_paths, start=1):
+        table_name = f"CSV{idx}"
+        
+        try:
+            print(f"📥 Loading {file_path} as {table_name}")
+            df = read_csv_from_s3(file_path)
+            
+            _registered_datasets[table_name] = df
+            
+            dataset_context = generate_dataset_context(df, table_name)
+            _dataset_context[table_name] = dataset_context
+            context_parts.append(f"\n{dataset_context}\n")
+            context_parts.append("="*70)
+            
+            print(f"✅ Loaded {table_name}: {len(df)} rows, {len(df.columns)} columns")
+            
+        except Exception as e:
+            print(f"❌ Error loading {file_path}: {str(e)}")
+            raise HTTPException(status_code=400, detail=f"Failed to load {file_path}: {str(e)}")
+    
+    full_context = "\n".join(context_parts)
+    return full_context
+
+# =====================================
+# 🔹 CHART.JS VISUALIZATION FUNCTION
+# =====================================
+# def generate_chartjs_visualization(result_df: pd.DataFrame, user_prompt: str) -> dict:
+#     """
+#     Generate Chart.js JSON config from DataFrame using LLM.
+    
+#     Args:
+#         result_df: pandas DataFrame to visualize
+#         user_prompt: User's visualization request
+        
+#     Returns:
+#         dict: Chart.js configuration JSON
+#     """
+#     # Initialize LLM
+#     llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)
+    
+#     # LLM Prompt
+#     prompt = f"""You are an expert data visualization analyst. Analyze the DataFrame and user query to generate the BEST Chart.js visualization.
+
+# User Query: "{user_prompt}"
+
+# Your Task:
+# 1. ANALYZE the DataFrame structure:
+#    - Identify column types (categorical, numerical, temporal)
+#    - Detect relationships between columns
+#    - Understand data distribution and patterns
+
+# 2. DETERMINE the optimal visualization:
+#    - For comparisons → bar chart
+#    - For trends over time → line chart
+#    - For proportions/parts of whole → pie/doughnut
+#    - For distributions → scatter/bubble
+#    - For multivariate comparisons → radar/polarArea
+#    - Match chart type to data characteristics AND user intent
+
+# 3. GENERATE Chart.js JSON with:
+#    - Correct axis mappings (categorical → labels, numerical → data)
+#    - Meaningful chart title derived from the data context
+#    - Appropriate colors for distinction
+#    - Professional styling options
+
+# Available chart types: bar, line, pie, doughnut, polarArea, radar, scatter, bubble
+
+# Color palette: #FF6384, #36A2EB, #FFCE56, #4BC0C0, #9966FF, #FF9F40
+
+# Output ONLY valid Chart.js JSON:
+# ```json
+# {{
+#   "type": "best_chart_type",
+#   "data": {{
+#     "labels": ["derived", "from", "dataframe"],
+#     "datasets": [{{
+#       "label": "Meaningful Label",
+#       "data": [actual, values, from, df],
+#       "backgroundColor": ["#FF6384", "#36A2EB", ...],
+#       "borderColor": ["#FF6384", "#36A2EB", ...],
+#       "borderWidth": 2
+#     }}]
+#   }},
+#   "options": {{
+#     "responsive": true,
+#     "maintainAspectRatio": false,
+#     "plugins": {{
+#       "title": {{
+#         "display": true,
+#         "text": "Descriptive Title Based on Data",
+#         "font": {{ "size": 16 }}
+#       }},
+#       "legend": {{
+#         "display": true,
+#         "position": "top"
+#       }}
+#     }},
+#     "scales": {{
+#       "y": {{
+#         "beginAtZero": true,
+#         "title": {{ "display": true, "text": "Y Axis Label" }}
+#       }},
+#       "x": {{
+#         "title": {{ "display": true, "text": "X Axis Label" }}
+#       }}
+#     }}
+#   }}
+# }}
+# ```"""
+    
+#     # Get response
+#     messages = [
+#         SystemMessage(content=prompt),
+#         HumanMessage(content=result_df.to_string())
+#     ]
+    
+#     response = llm.invoke(messages)
+    
+#     # Extract JSON
+#     json_match = re.search(r"```json\n(.*?)```", response.content, re.DOTALL)
+#     if not json_match:
+#         json_match = re.search(r"\{.*\}", response.content, re.DOTALL)
+    
+#     if not json_match:
+#         raise ValueError("No JSON found in LLM response")
+    
+#     config = json.loads(json_match.group(1 if "```json" in response.content else 0))
+#     vega_lite_path.append(config)
+#     print(f"Generated Chart.js Config: {json.dumps(config, indent=2)}")
+
+    
+#     # Validate
+#     if "type" not in config or "data" not in config:
+#         raise ValueError("Invalid Chart.js config")
+    
+#     return config
+def generate_vegalite_visualization(result_df: pd.DataFrame, user_prompt: str) -> dict:
+    """
+    Generate Vega-Lite JSON spec from DataFrame using LLM.
+    
+    Args:
+        result_df: pandas DataFrame to visualize
+        user_prompt: User's visualization request
+        
+    Returns:
+        dict: Vega-Lite specification JSON
+    """
+    # Initialize the LLM (using OpenAI's GPT-4o-mini as per the original)
+    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.3)
+    
+    # Construct the prompt, incorporating user query and DataFrame structure
+    prompt = f"""You are an expert data visualization analyst. Analyze the DataFrame and user query to generate the BEST Vega-Lite visualization.
+
+User Query: "{user_prompt}"
+
+Your Task:
+1. ANALYZE the DataFrame structure:
+   - Identify column types (categorical → nominal, numerical → quantitative, temporal)
+   - Detect relationships between columns
+   - Understand data distribution and patterns
+
+2. DETERMINE the optimal visualization:
+   - For comparisons → bar chart
+   - For trends over time → line chart
+   - For proportions/parts of whole → pie chart
+   - For distributions → scatter or point
+   - For multivariate comparisons → layered or concatenated views
+   - Match chart type to data characteristics AND user intent
+
+3. GENERATE Vega-Lite JSON with:
+   - Correct encoding mappings (e.g., x for categorical/nominal, y for numerical/quantitative, color for distinction)
+   - Data as inline JSON from the DataFrame
+   - Meaningful chart title derived from the user query and data context
+   - Appropriate colors for distinction using the provided palette
+   - Professional styling options, including tooltips for interactivity
+   - Reasonable chart dimensions (width: 400, height: 300)
+
+Available mark types: bar, line, area, pie, point (for scatter), circle (for bubble)
+
+Color palette: ["#FF6384", "#36A2EB", "#FFCE56", "#4BC0C0", "#9966FF", "#FF9F40"]
+
+**CRITICAL**: Output ONLY valid JSON with all property names enclosed in double quotes. Do NOT include any text, comments, or explanations outside the JSON. Ensure the JSON is complete and properly formatted for Vega-Lite v5 schema. All property names MUST be enclosed in double quotes.
+
+Example of valid output:
+{{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "data": {{
+    "values": [
+      {{"category": "A", "value": 10}},
+      {{"category": "B", "value": 20}},
+      {{"category": "C", "value": 30}}
+    ]
+  }},
+  "mark": "bar",
+  "encoding": {{
+    "x": {{"field": "category", "type": "nominal", "title": "Categories"}},
+    "y": {{"field": "value", "type": "quantitative", "title": "Values"}},
+    "color": {{"field": "category", "type": "nominal", "scale": {{"range": ["#FF6384", "#36A2EB", "#FFCE56"]}}}},
+    "tooltip": [
+      {{"field": "category", "type": "nominal", "title": "Category"}},
+      {{"field": "value", "type": "quantitative", "title": "Value"}}
+    ]
+  }},
+  "width": 400,
+  "height": 300,
+  "title": "Sample Bar Chart"
+}}
+
+Output ONLY the JSON content:
+{{
+  "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
+  "data": {{
+    "values": []
+  }},
+  "mark": "best_mark_type",
+  "encoding": {{
+    "x": {{"field": "derived_field", "type": "nominal", "title": "X Axis Label"}},
+    "y": {{"field": "derived_field", "type": "quantitative", "title": "Y Axis Label"}},
+    "color": {{"field": "derived_field", "type": "nominal", "scale": {{"range": ["#FF6384", "#36A2EB", "#FFCE56"]}}}},
+    "tooltip": [
+      {{"field": "derived_field", "type": "nominal", "title": "Label"}},
+      {{"field": "derived_field", "type": "quantitative", "title": "Value"}}
+    ]
+  }},
+  "width": 400,
+  "height": 300,
+  "title": "Descriptive Title Based on Data"
+}}
+"""
+    
+    # Convert DataFrame to string for LLM input
+    df_string = result_df.to_string()
+    
+    # Create messages for LLM
+    messages = [
+        SystemMessage(content=prompt),
+        HumanMessage(content=df_string)
+    ]
+    
+    # Invoke LLM to generate the Vega-Lite spec
+    try:
+        response = llm.invoke(messages)
+        print(f"Raw LLM Response:\n{response.content}\n")
+    except Exception as e:
+        raise ValueError(f"LLM invocation failed: {str(e)}")
+    
+    # Extract JSON block from LLM response
+    json_match = re.search(r"\{.*\}", response.content, re.DOTALL)
+    if not json_match:
+        raise ValueError(f"No valid JSON block found in LLM response:\n{response.content}")
+    
+    json_string = json_match.group(0).strip()
+    print(f"Extracted JSON String:\n{json_string}\n")
+    
+    # Parse JSON string into a Python dict
+    try:
+        config = json.loads(json_string)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON format: {str(e)}\nJSON String:\n{json_string}")
+    
+    # Log the generated config for debugging
+    print(f"Generated Vega-Lite Spec:\n{json.dumps(config, indent=2)}")
+    vega_lite_path.append(config)
+    
+    # Validate essential Vega-Lite properties
+    print(f"Generated Vega-Lite Config: {json.dumps(config, indent=2)}")
+
+    
+    # Validate
+    if "type" not in config or "data" not in config:
+        raise ValueError("Invalid Vega-Lite config")
+    
+    return config
+# =====================================
+# 🔹 VECTOR AGENT TOOL
+# =====================================
+def query_vector_agent_calling(user_query: str, collection_name: str) -> str:
+    """
+    Call the vector agent API to get relevant context for the given user query and collection name.
+    """
+    base_url = "https://srivatsavdamaraju-mvp-2-0-deploy-all-apis.hf.space/qdrant/search"
+    url = f"{base_url}?collection_name={collection_name}&mode=dense"
+    
+    headers = {
+        "accept": "application/json",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "query": user_query,
+        "top_k": 2
+    }
+
+    try:
+        response = requests.post(url, headers=headers, json=payload)
+        response.raise_for_status()
+
+        data = response.json()
+        results = data.get("results") or data.get("result") or data
+        if not results:
+            return "No relevant context found."
+
+        contexts = []
+        for item in results:
+            text = item.get("text") or item.get("payload", {}).get("text")
+            if text:
+                contexts.append(text)
+        
+        return "\n\n".join(contexts) if contexts else str(data)
+
+    except requests.RequestException as e:
+        print(f"Error calling vector agent API: {e}")
+        return "Error retrieving context."
+
+@function_tool
+async def vector_agent_tool(
+    user_query: Annotated[str, "The user's natural language question"],
+    collection_name: Annotated[str, "The name of the collection to search in"]
+) -> str:
+    """
+    Tool to call the vector agent API and retrieve relevant context.
+    
+    Args:
+        user_query: The user's natural language question.
+        collection_name: The name of the collection to search in.
+    
+    Returns:
+        A string containing the retrieved context.
+    """
+    return query_vector_agent_calling(user_query, collection_name)
+
+# =====================================
+# 🔹 SQL TOOL
+# =====================================
+@function_tool
+async def sql_tool(
+    sql_query: Annotated[str, "SQL query to execute on registered datasets"],
+    user_query: Annotated[str, "Original user question for visualization context"] = ""
+) -> SQLResult:
+    """
+    Execute SQL query on registered datasets and return results.
+    
+    Available tables: CSV1, CSV2, CSV3, etc. (based on loaded files)
+    Supported operations: SELECT, WITH (CTE)
+    
+    Args:
+        sql_query: The SQL query to execute
+        user_query: The original user question (REQUIRED for visualization context)
+    
+    Example queries:
+    - SELECT * FROM CSV1 WHERE age > 50 LIMIT 10
+    - SELECT a.name, b.value FROM CSV1 a JOIN CSV2 b ON a.id = b.id
+    
+    Example usage:
+    - sql_tool(
+        sql_query="SELECT * FROM CSV1",
+        user_query="What is the average age?"
+    )
+    """
+    global _registered_datasets, last_sql_query
+
+    last_sql_query = sql_query
+
+    start_time = datetime.now()
+    print(f"\n{'='*70}")
+    print(f"🔍 Executing SQL Query:\n{sql_query}\n")
+    print(f"📝 User Query Context: {user_query}\n")
+    print(f"{'='*70}\n")
+
+    try:
+        # Validate query
+        normalized_query = sql_query.strip().lower()
+        if not normalized_query.startswith(ALLOWED_SQL_KEYWORDS):
+            raise ValueError(
+                f"Only {', '.join(k.upper() for k in ALLOWED_SQL_KEYWORDS)} queries are allowed."
+            )
+        
+        if not _registered_datasets:
+            raise ValueError("No datasets loaded. Please provide file_paths first.")
+
+        # Execute query using DuckDB
+        conn = duckdb.connect(":memory:")
+        
+        for table_name, df in _registered_datasets.items():
+            conn.register(table_name, df)
+        
+        result_df = conn.execute(sql_query).df()
+        conn.close()
+        
+        if len(result_df) > MAX_RESULT_ROWS:
+            result_df = result_df.head(MAX_RESULT_ROWS)
+        
+        execution_time = (datetime.now() - start_time).total_seconds() * 1000
+        
+        summary_lines = [
+            f"Query returned {len(result_df)} rows with {len(result_df.columns)} columns: {', '.join(result_df.columns.tolist())}"
+        ]
+        
+        if not result_df.empty:
+            summary_lines.append("\nResults preview (first 10 rows):")
+            preview = result_df.head(10).to_string(index=False, max_rows=10)
+            summary_lines.append(preview)
+        
+        result_summary = "\n".join(summary_lines)
+        
+        print(f"✅ SQL executed: {len(result_df)} rows in {execution_time:.2f}ms\n")
+
+        return SQLResult(
+            sql_query=sql_query,
+            result_summary=result_summary,
+            columns=result_df.columns.tolist(),
+            row_count=len(result_df),
+            execution_time_ms=round(execution_time, 2),
+        )
+
+    except ValueError as ve:
+        raise ValueError(f"SQL validation error: {str(ve)}")
+    except Exception as e:
+        raise RuntimeError(f"SQL execution error: {str(e)}")
+
+# =====================================
+# 🔹 VISUALIZATION TASK
+# =====================================
+async def generate_visualization_task(query: str) -> Optional[dict]:
+    """Async task to generate Chart.js visualization using last_sql_query and upload to S3."""
+    global _registered_datasets, last_sql_query, vega_lite_path
+    
+    if not last_sql_query or not _registered_datasets:
+        print("⚠️ No SQL query executed or no datasets loaded, skipping visualization.")
+        return None
+
+    print(f"🎨 Generating Chart.js visualization...")
+    try:
+        # Re-execute the last SQL query to get the result DataFrame
+        conn = duckdb.connect(":memory:")
+        for table_name, df in _registered_datasets.items():
+            conn.register(table_name, df)
+        result_df = conn.execute(last_sql_query).df()
+        conn.close()
+
+        # Ensure user_query is a non-empty string
+        visualization_prompt = query.strip() if query and query.strip() else "Visualize the query results"
+        
+        print(f"🎯 User prompt for visualization: '{visualization_prompt}'")
+        print(f"🎯 User prompt type: {type(visualization_prompt)}")
+        print("+"*60)
+
+        # Generate Vega-Lite visualization
+        vega_lite_config = generate_vegalite_visualization(
+            result_df=result_df,
+            user_prompt=visualization_prompt
+        )
+        print(f"✅ Vega-Lite visualization generated.")
+        print("+"*60)
+        print(f"Vega-Lite Config: {json.dumps(vega_lite_config, indent=2)}")
+
+        return vega_lite_config
+        
+    except ValueError as ve:
+        raise ValueError(f"Visualization error: {str(ve)}")
+    except Exception as e:
+        raise RuntimeError(f"Visualization error: {str(e)}")
+# =====================================
+# 🔹 MAIN ANALYSIS FUNCTION
+# =====================================
+# async def main(query: str, file_paths: List[str] = None) -> str:
+#     """Main function to run data analysis with proper agent initialization."""
+    
+#     if file_paths is None:
+#         file_paths = []
+    
+#     print(f"\n{'='*70}")
+#     print(f"📋 Analysis Request")
+#     print(f"{'='*70}")
+#     print(f"Files: {', '.join(file_paths)}")
+#     print(f"Query: {query}")
+#     print(f"{'='*70}\n")
+    
+#     dataset_context = load_and_register_datasets(file_paths)
+    
+#     print(f"\n{'='*70}")
+#     print("📊 Dataset Context Summary")
+#     print(f"{'='*70}")
+#     print(dataset_context[:500] + "..." if len(dataset_context) > 500 else dataset_context)
+#     print(f"{'='*70}\n")
+
+#     general_purpose_agent = Agent(
+#         name="general_purpose_agent",
+#         instructions=(
+#             "You are a friendly AI assistant with high expertise in data analysis, SQL, and visualization. "
+#             "Only answer if the query is not related to data analysis."
+#         ),
+#         model="gpt-4o-mini",
+#         tools=[]
+#     )
+
+#     DataAnalyst_agent = Agent(
+#         name="DataAnalyst_agent",
+#         instructions=(
+#             f"""You are a professional data analyst with expertise in SQL and data analysis.
+
+# {dataset_context}
+
+# Your workflow:
+# 1. Analyze the user's question carefully
+# 2. Review the available datasets and their schemas above
+# 3. **Generate appropriate SQL query using sql_tool - ALWAYS pass the user_query parameter**
+   
+#    CRITICAL - Tool Usage:
+#    - Call sql_tool with BOTH parameters:
+#      * sql_query: Your SQL SELECT statement
+#      * user_query: The EXACT original user question (this creates the visualization)
+   
+#    Example correct usage:
+#    sql_tool(
+#        sql_query="SELECT age, COUNT(*) as count FROM CSV1 GROUP BY age ORDER BY age",
+#        user_query="Show me the age distribution of patients"
+#    )
+   
+#    - Use table names: CSV1, CSV2, CSV3, etc.
+#    - You can JOIN tables if needed
+#    - Use descriptive column aliases
+#    - Add ORDER BY for readability
+#    - Use LIMIT for large result sets
+   
+# 4. Interpret the results and provide clear, detailed insights with specific numbers and percentages
+
+# SQL Best Practices:
+# - Always check which columns exist in which tables
+# - Use table aliases for clarity (e.g., CSV1 AS c1)
+# - Filter data appropriately based on the question
+# - Aggregate data when asked for counts, sums, averages
+# - Sort results logically
+
+# Important:
+# - Only use columns that exist in the datasets above
+# - Reference the correct table names (CSV1, CSV2, etc.)
+# - **MANDATORY: Always pass both sql_query AND user_query to sql_tool**
+# - The user_query parameter is used to generate intelligent visualizations
+# - Always provide complete analysis with specific findings
+# - Include tables, numbers, and percentages in your response
+# - Provide insights in plain language for non-technical users
+#             """
+#         ),
+#         model="gpt-4o-mini",
+#         tools=[sql_tool],
+#     )
+
+#     orchestrator_agent = Agent(
+#         name="orchestrator_agent",
+#         instructions=(
+#             "You receive the user's query and route it to the DataAnalyst agent for execution. "
+#             "Always use the analyze_dataset tool to process data analysis queries. "
+#             "Pass the complete user query to the DataAnalyst agent."
+#         ),
+#         tools=[
+#             DataAnalyst_agent.as_tool(
+#                 tool_name="analyze_dataset",
+#                 tool_description="Analyze dataset, generate SQL, and visualize results."
+#             ),
+#             general_purpose_agent.as_tool(
+#                 tool_name="general_purpose_tool",
+#                 tool_description="Act like a human friendly AI assistant for non-data analysis queries."
+#             )
+#         ],
+#         model="gpt-4o-mini",
+#     )
+    
+#     print(f"\n{'='*70}")
+#     print("🚀 Running Orchestrator Agent")
+#     print(f"{'='*70}\n")
+    
+#     orchestrator_result = await Runner.run(orchestrator_agent, query)
+
+#     print(f"\n{'='*70}")
+#     print("📝 Orchestrator Output")
+#     print(f"{'='*70}")
+    
+#     analysis_text = ""
+#     for item in orchestrator_result.new_items:
+#         if hasattr(item, 'output') and item.output:
+#             analysis_text = item.output
+#             print(f"Found analysis output: {len(analysis_text)} characters")
+#         elif hasattr(item, 'raw_item') and hasattr(item.raw_item, 'content'):
+#             for content_item in item.raw_item.content:
+#                 if hasattr(content_item, 'text'):
+#                     analysis_text = content_item.text
+#                     print(f"Found message output: {len(analysis_text)} characters")
+    
+#     print(f"{'='*70}\n")
+
+#     synthesizer_agent = Agent(
+#         name="synthesizer_agent",
+#         instructions=(
+#             """You are a final report writer. Your job is to present the complete data analysis results.
+
+# CRITICAL RULES:
+# 1. Present the COMPLETE analysis from the DataAnalyst agent
+# 2. Include ALL tables, numbers, statistics, and findings
+# 3. Maintain all the detailed insights and percentages
+# 4. Use clear formatting with headers, tables, and bullet points
+# 5. DO NOT summarize or truncate the analysis
+# 6. DO NOT add generic closing statements like "feel free to ask"
+# 7. Make the report professional and easy to read
+
+# Your output should be the full, detailed analysis report with all findings intact.
+#             """
+#         ),
+#         model="gpt-4o-mini",
+#     )
+    
+#     print(f"\n{'='*70}")
+#     print("🎯 Running Synthesizer and Visualization in Parallel")
+#     print(f"{'='*70}\n")
+    
+#     synthesizer_task = asyncio.create_task(
+#         Runner.run(
+#             synthesizer_agent,
+#             f"Present this complete data analysis:\n\n{analysis_text}" if analysis_text else orchestrator_result.to_input_list()
+#         )
+#     )
+#     visualization_task = asyncio.create_task(generate_visualization_task(query))
+    
+#     synthesizer_result, visualization_result = await asyncio.gather(synthesizer_task, visualization_task, return_exceptions=True)
+    
+#     if isinstance(synthesizer_result, Exception):
+#         print(f"⚠️ Synthesizer task failed: {str(synthesizer_result)}")
+#         synthesizer_result = type('Result', (), {'final_output': 'Synthesizer task failed'})()
+    
+#     if isinstance(visualization_result, Exception):
+#         print(f"⚠️ Visualization task failed: {str(visualization_result)}")
+#         visualization_result = None
+
+#     print(f"\n{'='*70}")
+#     print("✅ FINAL ANSWER")
+#     print(f"{'='*70}")
+#     print(synthesizer_result.final_output)
+#     print(f"{'='*70}\n")
+    
+#     print(f"chartjs_config_path: {vega_lite_path}")
+#     print(f"{'='*70}\n")
+    
+#     final_api_output = synthesizer_result.final_output + f"\n\nChart.js Configuration saved at: {vega_lite_path[-1] if vega_lite_path else 'N/A'}"
+#     hidden_code = True if last_sql_query else False
+#     final_json_api_output = {
+#         "convoId": "1234",
+#         "type": "assistant_message",
+#         "hidden_code": hidden_code,
+#         "hidden_text": last_sql_query,
+#         "ai_message": synthesizer_result.final_output,
+#         "artifacts": vega_lite_path
+#     }
+#     print(f"Final JSON Output: {final_json_api_output}")
+#     return json.dumps(final_json_api_output)
+    # return final_json_api_output
+
+
+
+
+async def main(query: str, file_paths: List[str] = None) -> str:
+    """Main function to run data analysis with proper agent initialization and structured output."""
+    
+    if file_paths is None:
+        file_paths = []
+    
+    print(f"\n{'='*70}")
+    print(f"📋 Analysis Request")
+    print(f"{'='*70}")
+    print(f"Files: {', '.join(file_paths)}")
+    print(f"Query: {query}")
+    print(f"{'='*70}\n")
+    
+    dataset_context = load_and_register_datasets(file_paths)
+    
+    print(f"\n{'='*70}")
+    print("📊 Dataset Context Summary")
+    print(f"{'='*70}")
+    print(dataset_context[:500] + "..." if len(dataset_context) > 500 else dataset_context)
+    print(f"{'='*70}\n")
+
+    general_purpose_agent = Agent(
+        name="general_purpose_agent",
+        instructions=(
+            "You are a friendly AI assistant with high expertise in data analysis, SQL, and visualization. "
+            "Only answer if the query is not related to data analysis."
+        ),
+        model="gpt-4o-mini",
+        tools=[]
+    )
+
+    DataAnalyst_agent = Agent(
+        name="DataAnalyst_agent",
+        instructions=(
+            f"""You are a professional data analyst with expertise in SQL and data analysis.
+
+{dataset_context}
+
+Your workflow:
+1. Analyze the user's question carefully
+2. Review the available datasets and their schemas above
+3. **Generate appropriate SQL query using sql_tool - ALWAYS pass the user_query parameter**
+   
+   CRITICAL - Tool Usage:
+   - Call sql_tool with BOTH parameters:
+     * sql_query: Your SQL SELECT statement
+     * user_query: The EXACT original user question (this creates the visualization)
+   
+   Example correct usage:
+   sql_tool(
+       sql_query="SELECT age, COUNT(*) as count FROM CSV1 GROUP BY age ORDER BY age",
+       user_query="Show me the age distribution of patients"
+   )
+   
+   - Use table names: CSV1, CSV2, CSV3, etc.
+   - You can JOIN tables if needed
+   - Use descriptive column aliases
+   - Add ORDER BY for readability
+   - Use LIMIT for large result sets
+   
+4. Interpret the results and provide clear, detailed insights with specific numbers and percentages
+
+SQL Best Practices:
+- Always check which columns exist in which tables
+- Use table aliases for clarity (e.g., CSV1 AS c1)
+- Filter data appropriately based on the question
+- Aggregate data when asked for counts, sums, averages
+- Sort results logically
+
+Important:
+- Only use columns that exist in the datasets above
+- Reference the correct table names (CSV1, CSV2, etc.)
+- **MANDATORY: Always pass both sql_query AND user_query to sql_tool**
+- The user_query parameter is used to generate intelligent visualizations
+- Always provide complete analysis with specific findings
+- Include tables, numbers, and percentages in your response
+- Provide insights in plain language for non-technical users
+            """
+        ),
+        model="gpt-4o-mini",
+        tools=[sql_tool],
+    )
+
+    orchestrator_agent = Agent(
+        name="orchestrator_agent",
+        instructions=(
+            "You receive the user's query and route it to the DataAnalyst agent for execution. "
+            "Always use the analyze_dataset tool to process data analysis queries. "
+            "Pass the complete user query to the DataAnalyst agent."
+        ),
+        tools=[
+            DataAnalyst_agent.as_tool(
+                tool_name="analyze_dataset",
+                tool_description="Analyze dataset, generate SQL, and visualize results."
+            ),
+            general_purpose_agent.as_tool(
+                tool_name="general_purpose_tool",
+                tool_description="Act like a human friendly AI assistant for non-data analysis queries."
+            )
+        ],
+        model="gpt-4o-mini",
+    )
+    
+    print(f"\n{'='*70}")
+    print("🚀 Running Orchestrator Agent")
+    print(f"{'='*70}\n")
+    
+    orchestrator_result = await Runner.run(orchestrator_agent, query)
+
+    print(f"\n{'='*70}")
+    print("📝 Orchestrator Output")
+    print(f"{'='*70}")
+    
+    analysis_text = ""
+    for item in orchestrator_result.new_items:
+        if hasattr(item, 'output') and item.output:
+            analysis_text = item.output
+            print(f"Found analysis output: {len(analysis_text)} characters")
+        elif hasattr(item, 'raw_item') and hasattr(item.raw_item, 'content'):
+            for content_item in item.raw_item.content:
+                if hasattr(content_item, 'text'):
+                    analysis_text = content_item.text
+                    print(f"Found message output: {len(analysis_text)} characters")
+    
+    print(f"{'='*70}\n")
+
+    synthesizer_agent = Agent(
+        name="synthesizer_agent",
+        instructions=(
+            f"""You are a final report writer. Your job is to present the complete data analysis results and suggest follow-up queries.
+
+CRITICAL RULES:
+1. Present the COMPLETE analysis from the DataAnalyst agent
+2. Include ALL tables, numbers, statistics, and findings
+3. Maintain all the detailed insights and percentages
+4. Use clear formatting with headers, tables, and bullet points
+5. DO NOT summarize or truncate the analysis
+6. DO NOT add generic closing statements like "feel free to ask"
+7. Generate a 'Suggested Next Queries' section with at least 4 follow-up queries
+   - Each query should include:
+     - query: A natural language query
+     - description: A brief explanation of what the query explores
+     - recommended_chart_type: A suitable chart type (bar, line, pie, point, area, circle)
+8. Make the report professional and easy to read
+
+Dataset Context for Reference:
+{dataset_context}
+
+Your output should be a professional, detailed analysis report with all findings intact, followed by suggested next queries.
+            """
+        ),
+        model="gpt-4o-mini",
+    )
+    
+    print(f"\n{'='*70}")
+    print("🎯 Running Synthesizer and Visualization in Parallel")
+    print(f"{'='*70}\n")
+    
+    synthesizer_task = asyncio.create_task(
+        Runner.run(
+            synthesizer_agent,
+            f"Present this complete data analysis:\n\n{analysis_text}" if analysis_text else orchestrator_result.to_input_list()
+        )
+    )
+    visualization_task = asyncio.create_task(generate_visualization_task(query))
+    
+    synthesizer_result, visualization_result = await asyncio.gather(synthesizer_task, visualization_task, return_exceptions=True)
+    
+    if isinstance(synthesizer_result, Exception):
+        print(f"⚠️ Synthesizer task failed: {str(synthesizer_result)}")
+        synthesizer_result = type('Result', (), {'final_output': 'Synthesizer task failed'})()
+    
+    if isinstance(visualization_result, Exception):
+        print(f"⚠️ Visualization task failed: {str(visualization_result)}")
+        visualization_result = None
+
+    # Extract suggested next queries from synthesizer output
+    # next_queries = []
+    # if synthesizer_result.final_output:
+    #     # Use regex to extract the Suggested Next Queries section
+    #     next_queries_match = re.search(r"## Suggested Next Queries\n\n(.*?)(?:\n\n|$)", synthesizer_result.final_output, re.DOTALL)
+    #     if next_queries_match:
+    #         queries_text = next_queries_match.group(1).strip()
+    #         # Split by list items and parse
+    #         query_items = queries_text.split('\n- ')
+    #         for item in query_items:
+    #             if not item.strip():
+    #                 continue
+    #             # Extract query, description, and chart type using regex
+    #             query_match = re.search(r"Query: (.*?); Description: (.*?); Recommended Chart Type: (\w+)", item.strip())
+    #             if query_match:
+    #                 next_queries.append({
+    #                     "query": query_match.group(1).strip(),
+    #                     "description": query_match.group(2).strip(),
+    #                     "recommended_chart_type": query_match.group(3).strip()
+    #                 })
+
+    # Extract suggested next queries from synthesizer output
+    # Extract suggested next queries from synthesizer output
+    next_queries = []
+    if synthesizer_result.final_output:
+        # Capture everything under "## Suggested Next Queries"
+        match = re.search(
+            r"## Suggested Next Queries\s*(.*?)(?=\n## |\Z)",
+            synthesizer_result.final_output,
+            re.DOTALL
+        )
+
+        if match:
+            section = match.group(1).strip()
+
+            # Split by either "### Query" or numbered "1. **Query:**" formats
+            query_blocks = re.split(r"(?:###\s*Query\s*\d+|^\d+\.\s+\*\*Query:\*\*)", section, flags=re.MULTILINE)
+
+            for block in query_blocks[1:]:  # skip header/non-query text
+                block = block.strip()
+                if not block:
+                    continue
+
+                # Extract fields (support both markdown and colon formats)
+                query_match = re.search(r"\*\*Query[:\*]*\s*\**\s*(.*)", block)
+                desc_match = re.search(r"-\s*\*\*Description[:\*]*\s*\**\s*(.*)", block)
+                chart_match = re.search(r"-\s*\*\*Recommended Chart Type[:\*]*\s*\**\s*(.*)", block)
+
+                # Also handle colon-only versions (like in your example)
+                if not query_match:
+                    query_match = re.search(r"[:\-]\s*\*\*Query[:\*]*\s*\**\s*(.*)", block)
+                if not desc_match:
+                    desc_match = re.search(r"[:\-]\s*\*\*description\*\*[:\-]*\s*(.*)", block, re.IGNORECASE)
+                if not chart_match:
+                    chart_match = re.search(r"[:\-]\s*\*\*recommended_chart_type\*\*[:\-]*\s*(.*)", block, re.IGNORECASE)
+
+                next_queries.append({
+                    "query": query_match.group(1).strip() if query_match else None,
+                    "description": desc_match.group(1).strip() if desc_match else None,
+                    "recommended_chart_type": chart_match.group(1).strip() if chart_match else None
+                })
+
+        # Step 5: (Optional) Pretty print to confirm
+        
+    # Prepare final JSON output
+    final_json_api_output = {
+        "convoId": "1234",  # Hardcoded as per example
+        "type": "assistant_message",
+        "hidden_code": bool(last_sql_query),
+        "hidden_text": last_sql_query if last_sql_query else "",
+        "ai_message": synthesizer_result.final_output,
+        "artifacts": vega_lite_path,
+        "next_queries": next_queries
+    }
+
+    print(f"\n{'='*70}")
+    print("✅ FINAL ANSWER")
+    print(f"{'='*70}")
+    print(json.dumps(final_json_api_output, indent=2))
+    print(f"{'='*70}\n")
+    
+    return json.dumps(final_json_api_output)
+# =====================================
+# 🔹 STANDALONE TESTING
+

Routes/helpers/main_agent_helpers.py

@@ -0,0 +1,255 @@
+import pandas as pd
+from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain_openai import ChatOpenAI
+import os
+import dotenv
+from agents import Agent, Runner, function_tool ,trace
+from agents.extensions.memory import RedisSession
+import asyncio
+from .autovis_tool import run_autoviz
+
+# Load environment variables
+dotenv.load_dotenv()
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+#redis_url 
+Redis_url = os.getenv("REDIS_URL")
+
+
+# # Load CSV
+# csv_path = r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+# df = pd.read_csv(csv_path)
+
+
+data = {
+    'Employee_ID': [101, 102, 103, 104, 105, 106],
+    'Name': ['Alice Johnson', 'Bob Smith', 'Charlie Brown', 'Diana Prince', 'Eve Adams', 'Frank Miller'],
+    'Department': ['Sales', 'IT', 'Marketing', 'Sales', 'IT', 'Finance'],
+    'Hire_Date': pd.to_datetime(['2021-05-15', '2020-11-01', '2022-01-20', '2021-08-10', '2023-03-01', '2020-07-25']),
+    'Salary': [70000, 95000, 60000, 75000, 88000, 110000],
+    'Performance_Score': [4.2, 4.8, 3.5, 4.0, 4.5, 3.9],
+    'Is_Manager': [False, True, False, False, True, True]
+}
+
+df = pd.DataFrame(data)
+# Initialize LLM
+llm = ChatOpenAI(temperature=0)
+
+# Create Pandas agent
+pandas_agent = create_pandas_dataframe_agent(llm, df, verbose=True, allow_dangerous_code=True)
+
+# Wrap it as a tool
+@function_tool
+def df_agent(query: str):
+    """Query the healthcare dataset using natural language."""
+    response = pandas_agent.invoke(query)
+    return response
+
+
+@function_tool
+
+def autoviz_tool():
+    """Run AutoViz on the given CSV file and return the plot directory."""
+    response = run_autoviz(filename=csv_path,chart_format="html")
+    print("+"*60)
+    return response
+    print("+"*60)
+# autoviz_tool()
+
+
+DataAnalyst_agent = Agent(
+    name="DataAnalyst_agent",
+    instructions=(
+        "You are a data analyst given a dataframe tool and autoviz tool. "
+        "Always use these tools to answer any question about the data. "
+        
+    ),
+    model="gpt-4o-mini",
+    tools=[df_agent],   
+)
+Business_Intelligence_Agent = Agent(
+    name="Business Intelligence Agent",
+    instructions="You are a business intelligence agent and you are given a dataframe tool and you need to analyse it and give some insights on it",
+    model="gpt-4o-mini",
+    tools=[df_agent],
+)
+
+Data_Scientist_Agent = Agent(
+    name="Data Scientist Agent",
+    instructions="You are a data scientist and you are given a dataframe tool and you need to analyse it and give some insights on it",
+    model="gpt-4o-mini",
+    tools=[df_agent],
+)
+
+autoviz_agent = Agent(
+    name="autoviz_agent",
+    instructions="You are a data scientist and you use onlt autoviz tool and you need to vizualise data",
+    model="gpt-4o-mini",
+    tools=[autoviz_tool],
+)
+
+
+
+#Rephraser nlp agent 
+rephraser_agent = Agent(
+    name="rephraser_agent",
+    instructions="You rephrase the user's message to make it more natural and fluent",
+    handoff_description="An english to rephraser",
+    model="gpt-4o-mini"
+)
+
+# --- Define Orchestrator Agent ---
+# orchestrator_agent = Agent(
+#     name="orchestrator_agent",
+#     instructions=(
+#         "You are a data analysis orchestrator. You receive a user's query and use the df_agent "
+#         "tool to analyze the healthcare dataset and provide an insightful response."
+#         "You never analyze the data on your own — always use df_agent."
+#     ),
+#     tools=[
+#         DataAnalyst_agent.as_tool(
+#             tool_name="analyze_dataframe",
+#             tool_description="Analyze the healthcare dataset based on user queries."
+#         ),
+#          rephraser_agent.as_tool(
+#             tool_name="rephrase",
+#             tool_description="Rephrase the user's message to make it more natural and fluent",
+#         ),
+#         Business_Intelligence_Agent.as_tool(
+#             tool_name="Business_Intelligence_Agent",
+#             tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+#         ),
+#         Data_Scientist_Agent.as_tool(
+#             tool_name="DataScientist_Agent",
+#             tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+#         ),
+#     ],
+# )
+
+orchestrator_agent = Agent(
+    name="orchestrator_agent",
+    instructions=(
+        "You are a data analysis orchestrator. You receive a user's query and use the df_agent "
+        "tool to analyze the healthcare dataset and provide an insightful response."
+        "You never analyze the data on your own — always use df_agent."
+        "you always use autoviz_agent to only visualize the data.if user asked about plots , or vizualization then use autoviz_agent "
+    ),
+    tools=[
+        DataAnalyst_agent.as_tool(
+            tool_name="analyze_dataframe",
+            tool_description="Analyze the healthcare dataset based on user queries."
+        ),
+        rephraser_agent.as_tool(
+            tool_name="rephrase",
+            tool_description="Rephrase the user's message to make it more natural and fluent",
+        ),
+        Business_Intelligence_Agent.as_tool(
+            tool_name="Business_Intelligence_Agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+        Data_Scientist_Agent.as_tool(
+            tool_name="DataScientist_Agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+        autoviz_agent.as_tool(
+            tool_name="autoviz_agent",
+            tool_description="use this only for plots and vizualization",
+        ),
+
+    ],
+)
+
+
+# --- Define Synthesizer Agent ---
+synthesizer_agent = Agent(
+    name="synthesizer_agent",
+    instructions=(
+        "You review the dataframe analysis results, clean up the language if needed, "
+        "and present a clear final answer to the user."
+    ),
+)
+
+
+# # --- Main Async Runner ---
+# #without_streamming===================================
+# async def main(query):
+#     #
+
+#     # Run orchestrator
+#     orchestrator_result = await Runner.run(orchestrator_agent, query)
+
+#     for item in orchestrator_result.new_items:
+#         print(f"Intermediate result: {item}")
+
+#     # Run synthesizer
+#     synthesizer_result = await Runner.run(
+#         synthesizer_agent, orchestrator_result.to_input_list()
+#     )
+
+#     print(f"\n\nFinal Answer:\n{synthesizer_result.final_output}")
+#     return synthesizer_result.final_output
+
+#=========with_redis_session_memory_stort_term=================
+async def main(query):
+    user_id = "8919614347"
+    session_id = "123456789"
+
+    print("Creating session...")
+    session = RedisSession.from_url(
+        session_id,
+        url=Redis_url,
+        key_prefix=f"{user_id}:",
+    )
+    #
+
+    # Run orchestrator
+    orchestrator_result = await Runner.run(orchestrator_agent, query, session=session) #  session=session
+
+    for item in orchestrator_result.new_items:
+        print(f"Intermediate result: {item}")
+
+    # Run synthesizer
+    synthesizer_result = await Runner.run(
+        synthesizer_agent, orchestrator_result.to_input_list()
+    )
+
+    print(f"\n\nFinal Answer:\n{synthesizer_result.final_output}")
+    return synthesizer_result.final_output
+
+#with_streamming
+# --- Main Async Function with Streaming + Trace ---
+# async def main(query: str):
+#     with trace("Dataframe analysis trace"):
+#         print("Analyzing your query...\n")
+
+#         # Streamed orchestration
+#         result = Runner.run_streamed(orchestrator_agent, query)
+
+#         async for event in result.stream_events():
+#             if (
+#                 event.type == "run_item_stream_event"
+#                 and event.item.type == "tool_call_item"
+#             ):
+#                 print(f"\n🔧 Tool Call: {event.item.name}")
+#             elif (
+#                 event.type == "run_item_stream_event"
+#                 and event.item.type == "message_output_item"
+#             ):
+#                 print(f"🧠 Model Output: {event.item.raw_item.output_text}")
+#             elif event.type == "run_step_stream_event":
+#                 print(f"Step Event: {event.step_name}")
+#             else:
+#                 print(f"Other Event: {event.type}")
+
+#         print("\nSynthesizing final output...\n")
+
+#         # Run the synthesizer for final cleanup
+#         synthesizer_result = await Runner.run(
+#             synthesizer_agent, result.to_input_list()
+#         )
+
+#         print(f"\n✅ Final Answer:\n{synthesizer_result.final_output}")
+
+# if __name__ == "__main__":
+#     user_query = input("Enter the query: ")
+#     asyncio.run(main(user_query))

Routes/helpers/pandas_ai_agent.py

@@ -0,0 +1,39 @@
+import pandas as pd
+from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain_openai import ChatOpenAI
+import os 
+import dotenv
+import asyncio
+
+from agents import Agent, ItemHelpers, MessageOutputItem, Runner, trace
+
+
+from agents import Agent, Runner, function_tool ,CodeInterpreterTool , trace
+import asyncio
+import os
+import dotenv
+dotenv.load_dotenv()
+
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+
+csv_path = r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+
+# Load sample DataFrame
+df = pd.read_csv(csv_path)
+
+# Initialize LLM
+llm = ChatOpenAI(temperature=0)
+
+# Create the agent with the DataFrame
+agent = create_pandas_dataframe_agent(llm, df, verbose=True ,allow_dangerous_code=True)
+
+# Ask a question
+# response = agent.invoke("How many people are older than 28?")
+# print(response)
+def df_agent(query: str):
+    response = agent.invoke(query)
+    return response
+
+
+ 

Routes/helpers/plotly_langchain_agent.py

@@ -0,0 +1,207 @@
+import pandas as pd
+from langchain_openai import ChatOpenAI
+from langchain_experimental.agents import create_pandas_dataframe_agent
+from langchain.tools import Tool
+from typing import Any, Dict, Optional
+import os
+import uuid
+import json
+import plotly.io as pio
+import plotly.graph_objects as go
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from starlette.responses import FileResponse, JSONResponse
+
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv() 
+
+# ----------------- Configuration & Data Loading -----------------
+# Now, retrieve the key from the environment (loaded from .env)
+openai_key = os.getenv("OPENAI_API_KEY") 
+os.environ["OPENAI_API_KEY"] = openai_key 
+
+# # ... rest of your code ...
+
+# # --- 1. CONFIGURATION & DATA LOADING ---
+# # NOTE: In a production environment, file paths should be configurable and
+# # API keys should be handled securely (e.g., using FastAPI secrets/dependencies).
+
+# # 🚨 IMPORTANT: Replace with your actual key before running!
+# os.environ["OPENAI_API_KEY"] = "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"
+
+# Define a persistent directory for chart outputs
+CHART_DIR = "charts"
+os.makedirs(CHART_DIR, exist_ok=True)
+
+# Initialize DataFrames
+df1 = pd.DataFrame()
+df2 = pd.DataFrame()
+
+# Load DataFrames (adjust paths as necessary for your local environment)
+# IMPORTANT: Since this is a FastAPI app, these paths must be local paths accessible to the server.
+# The original /content/ paths from Colab are generally not valid here.
+csv_file_path = "Visadataset.csv" 
+excel_file_path = "HRanalytics.xlsx"
+
+try:
+    df1 = pd.read_csv(csv_file_path)
+    print(f"Dataframe df1 (Visa Dataset) loaded from: {csv_file_path}")
+except FileNotFoundError:
+    print(f"Error: CSV file at {csv_file_path} not found. df1 remains empty. Using empty DataFrame.")
+
+try:
+    df2 = pd.read_excel(excel_file_path)
+    print(f"Dataframe df2 (HR Analytics) loaded from: {excel_file_path}")
+except FileNotFoundError:
+    print(f"Error: Excel file at {excel_file_path} not found. df2 remains empty. Using empty DataFrame.")
+
+# --- 2. VISUALIZATION TOOL DEFINITION (Modified for API Environment) ---
+
+def plotly_visualization_tool(code: str) -> str:
+    """
+    Executes Python code to generate a Plotly figure, saves it as an HTML file 
+    in the CHART_DIR, and returns the **full file path**.
+
+    Args:
+        code: A string containing the complete Python code for Plotly visualization.
+
+    Returns:
+        A string containing the full path to the generated HTML file, or an error message.
+    """
+    exec_globals = {
+        'go': go,
+        'pio': pio,
+        'pd': pd,
+        'df1': df1,
+        'df2': df2
+    }
+    figure_obj = None
+
+    try:
+        # 1. Execute the code generated by the agent
+        exec(code, exec_globals) 
+        figure_obj = exec_globals.get('fig')
+
+        if not figure_obj or not isinstance(figure_obj, go.Figure):
+            return "ERROR: Visualization code executed, but no valid Plotly Figure object (go.Figure) was found in the 'fig' variable."
+
+    except Exception as e:
+        return f"ERROR during code execution for visualization: {e}"
+
+    # 2. Save the figure to a file and return the path
+    try:
+        filename = f"plotly_chart_{uuid.uuid4().hex[:8]}.html"
+        # Save to the local CHART_DIR
+        filepath = os.path.join(CHART_DIR, filename) 
+
+        # Save the HTML file to disk
+        pio.write_html(
+            figure_obj,
+            file=filepath,
+            include_plotlyjs='cdn',
+            auto_open=False
+        )
+
+        # *** REVISED RETURN ***: Return the relative path for easy API access
+        # The user will download the file using a separate endpoint.
+        return filename
+
+    except Exception as e:
+        return f"ERROR saving HTML file to path: {e}"
+
+# --- 3. AGENT SETUP ---
+
+plotly_tool = Tool(
+    name="plotly_visualization",
+    func=plotly_visualization_tool,
+    description=(
+        "Use this tool ONLY when the user explicitly asks for a chart, plot, or visualization. "
+        "The tool saves the chart as an HTML file and returns the **filename only** (e.g., plotly_chart_xxxx.html). "
+        "The input MUST be a COMPLETE and self-contained Python code string using 'plotly.graph_objects as go' "
+        "to generate the requested chart using DataFrames 'df1' (Visadataset) or 'df2' (HRanalytics). "
+        "The final figure object MUST be assigned to the variable 'fig' and the code MUST NOT contain fig.show(). "
+        "Example input: \"import plotly.graph_objects as go\\nfig = go.Figure([go.Bar(x=df1['case_status'].value_counts().index, y=df1['case_status'].value_counts().values)])\\nfig.update_layout(title='Visa Status Count', xaxis_title='Status', yaxis_title='Count')\""
+    ),
+    return_direct=False
+)
+
+llm = ChatOpenAI(
+    model="gpt-4o-mini",
+    api_key=os.environ["OPENAI_API_KEY"],
+    temperature=0.3,
+    timeout=60, # Increased timeout for potential long agent/tool calls
+    max_retries=3
+)
+
+# Pass both DataFrames (df1, df2) to the agent
+agent = create_pandas_dataframe_agent(
+    llm=llm,
+    df=[df1, df2],
+    verbose=True,
+    allow_dangerous_code=True,
+    agent_type="tool-calling",
+    extra_tools=[plotly_tool]
+)
+
+# --- 4. FASTAPI APPLICATION ---
+
+app = FastAPI(title="Pandas Agent API with Plotly Visualization")
+
+class QueryRequest(BaseModel):
+    question: str
+
+class QueryResponse(BaseModel):
+    message: str
+    chart_filename: Optional[str] = None
+
+@app.get("/")
+def read_root():
+    return {"message": "Pandas Agent API is running. Use the /query endpoint to ask questions."}
+
+@app.post("/query", response_model=QueryResponse)
+async def handle_query(request: QueryRequest):
+    """
+    Handles a user query by invoking the LangChain agent.
+    If a chart is generated, it returns the filename for download.
+    """
+    question = request.question
+    
+    try:
+        response = agent.invoke({"input": question})
+        output = response.get("output", str(response))
+
+        # Check if the output is a filename returned by the tool (e.g., plotly_chart_xxxx.html)
+        if output.startswith("plotly_chart_") and output.endswith(".html"):
+            filename = output
+            return QueryResponse(
+                message=f"Agent generated a chart. Download the file using the /download/{filename} endpoint.",
+                chart_filename=filename
+            )
+        
+        # If no chart was generated, return the text output
+        return QueryResponse(message=output)
+
+    except Exception as e:
+        # Log the error internally
+        print(f"Error during agent invocation: {e}")
+        raise HTTPException(status_code=500, detail=f"Internal server error: {e}")
+
+@app.get("/download/{filename}")
+async def download_chart(filename: str):
+    """
+    Serves the generated Plotly HTML file for download.
+    """
+    filepath = os.path.join(CHART_DIR, filename)
+    
+    if not filename.startswith("plotly_chart_") or ".." in filename:
+        raise HTTPException(status_code=400, detail="Invalid filename format.")
+
+    if not os.path.exists(filepath):
+        raise HTTPException(status_code=404, detail="Chart file not found.")
+
+    return FileResponse(filepath, filename=filename, media_type="text/html")
+
+# --- END OF FASTAPI CODE ---

Routes/helpers/report_generation_helpers.py

@@ -0,0 +1,242 @@
+# from google import genai
+# from datetime import datetime
+# import requests
+# import base64
+
+
+
+
+# gemini_api_key ="AIzaSyAAAUn6xgLwzCkdUkVViaYnLo40pVN0ABc"
+
+# # Set your Gemini AI API key
+# client = genai.Client(api_key=gemini_api_key)
+# generation_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+# # Context and executive summary
+# context = """
+# Sales data for Q1 2025 shows product A had a 30% increase in revenue,
+# while product B remained flat. Marketing campaigns improved customer engagement.
+# """
+
+# executive_summary = """
+# This report highlights quarterly sales performance.
+# Product A shows significant growth, while Product B remains stable.
+# Includes static CSS-only charts and tables for visual analysis.
+# """
+
+# # Prompt to generate HTML + CSS
+# prompt = f"""
+# Generate a  best design in css static HTML report (only HTML and CSS, no JS) including:
+# - Executive summary
+# - Static graphs (bar or line charts using CSS)
+# - Tables for data visualization if needed
+# - Professional layout and styling
+
+# Executive Summary:
+# {executive_summary}
+
+# Context:
+# {context}
+# """
+
+# # # Generate content using Gemini AI
+# # response = client.models.generate_content(
+# #     model="gemini-2.5-flash",
+# #     contents=prompt
+    
+# # )
+
+# # # Get the generated HTML
+# # html_code = response.text
+
+
+# # print(html_code)
+
+# # # Save to file
+# # with open("static_report.html", "w", encoding="utf-8") as f:
+# #     f.write(html_code)
+
+# # print("Static HTML report generated: static_report.html")
+
+# # this is for the production grade report generation
+# #def return_html_report(format_type:str,report_name:str,include_citations:bool,success:bool,list_of_queries:list[str],theme:str,pageoptions:dict):
+
+# def return_html_report():
+#     # Generate content using Gemini AI
+#     response = client.models.generate_content(
+#         model="gemini-2.5-flash",
+#         contents=prompt
+        
+#     )
+
+#     # Get the generated HTML
+#     html_code = response.text
+    
+# # Find the index where the DOCTYPE starts
+#     start_index = html_code.find("<!DOCTYPE html>")
+
+# # Extract from that point forward
+#     cleaned_html = html_code[start_index:]
+
+#     print(cleaned_html)
+
+
+#     print(cleaned_html)
+
+#     # Save to file
+#     # with open("static_report.html", "w", encoding="utf-8") as f:
+#     #     f.write(html_code)
+
+#     # print("Static HTML report generated: static_report.html")
+#     return cleaned_html
+
+# # html_content = return_html_report()
+# # print(html_content)
+
+
+# import requests
+# import base64
+# # 
+# # url = "https://srivatsavdamaraju-htmlnode.hf.space/api/html-to-pdf"
+# # payload = {
+# #     "html_content": html_content,
+# #     "pdf_options": {
+# #         "format": "A4"
+# #     }
+# # }
+
+# # response = requests.post(url, json=payload)
+
+# # result = response.json()
+# # print("="*550)
+# # print(result)
+# # print("="*550)
+
+# # # Save PDF to file
+# # with open("output.pdf", "wb") as f:
+# #     f.write(base64.b64decode(result["pdf_base64"]))
+# def html_to_pdf_via_api(html_content):
+#     url = "https://srivatsavdamaraju-htmlnode.hf.space/api/html-to-pdf"
+#     payload = {
+#         "html_content": html_content,
+#         "pdf_options": {
+#             "format": "A4"
+#         }
+#     }
+
+#     response = requests.post(url, json=payload)
+#     result = response.json()
+#     print(result)
+#     return result
+
+
+
+# def html_to_pdf_via_api_with_pdf(html_content, output_pdf_path):
+#     url = "https://srivatsavdamaraju-htmlnode.hf.space/api/html-to-pdf"
+#     payload = {
+#         "html_content": html_content,
+#         "pdf_options": {
+#             "format": "A4"
+#         }
+#     }
+
+#     response = requests.post(url, json=payload)
+#     result = response.json()
+#     print(result)
+
+#     # Save PDF to file
+#     # with open(output_pdf_path, "wb") as f:
+#     #     f.write(base64.b64decode(result["pdf_base64"]))
+
+
+# # if __name__ == "__main__":
+    
+# #     html_to_pdf_via_api_with_pdf(html_content, "generated.pdf")
+# #     print("PDF report generated: generated.pdf")
+
+
+
+from google import genai
+from datetime import datetime
+import requests
+import base64
+
+# Initialize Gemini client
+gemini_api_key ="AIzaSyAAAUn6xgLwzCkdUkVViaYnLo40pVN0ABc"
+client = genai.Client(api_key=gemini_api_key)
+
+def generate_prompt(context, executive_summary):
+    """Generate the text prompt for Gemini."""
+    return f"""
+    Generate a professional static HTML report (only HTML and CSS, no JS) including:
+    - Executive summary
+    - Static CSS-based graphs or tables
+    - Elegant, professional styling
+    
+    Executive Summary:
+    {executive_summary}
+
+    Context:
+    {context}
+    """
+
+def return_html_report(context=None, executive_summary=None):
+    """Generate HTML report using Gemini API."""
+    if not context:
+        context = "Default context: Sales increased by 10%."
+    if not executive_summary:
+        executive_summary = "Default summary: Q1 growth observed."
+
+    prompt = generate_prompt(context, executive_summary)
+    
+    response = client.models.generate_content(
+        model="gemini-2.5-flash",
+        contents=prompt
+    )
+    
+    html_code = response.text
+    start_index = html_code.find("<!DOCTYPE html>")
+    cleaned_html = html_code[start_index:] if start_index != -1 else html_code
+    print(cleaned_html)
+    return cleaned_html
+
+# return_html_report()
+
+
+def html_to_pdf_via_api(html_content):
+    """Send HTML to your Hugging Face PDF API."""
+    url = "https://srivatsavdamaraju-htmlnode.hf.space/api/html-to-pdf"
+    payload = {
+        "html_content": html_content,
+        "pdf_options": {"format": "A4"}
+    }
+
+    response = requests.post(url, json=payload)
+    result = response.json()
+    return result
+
+
+def generate_report_files(format_type="html", regenerate_flags={"html": True, "pdf": True}):
+    """
+    Generate report based on flags to avoid redundant work.
+    - If regenerate_flags['html'] is False, skip HTML generation.
+    - If regenerate_flags['pdf'] is False, skip PDF conversion.
+    """
+    html_content = None
+    pdf_result = None
+
+    # Generate HTML only if needed
+    if regenerate_flags.get("html", True):
+        html_content = return_html_report()
+    else:
+        print("Skipping HTML generation (flag set to False)")
+
+    # Generate PDF only if needed
+    if format_type == "pdf" and regenerate_flags.get("pdf", True):
+        if not html_content:
+            html_content = return_html_report()
+        pdf_result = html_to_pdf_via_api(html_content)
+    elif format_type == "pdf":
+        print("Skipping PDF generation (flag set to False)")
+
+    return {"html": html_content, "pdf": pdf_result}

Routes/helpers/rough.py

@@ -0,0 +1,161 @@
+import pandas as pd
+from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain_openai import ChatOpenAI
+import os
+import dotenv
+from agents import Agent, Runner, function_tool ,trace
+import asyncio
+
+# Load environment variables
+dotenv.load_dotenv()
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+# Load CSV
+csv_path = r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+df = pd.read_csv(csv_path)
+
+# Initialize LLM
+llm = ChatOpenAI(temperature=0)
+
+# Create Pandas agent
+pandas_agent = create_pandas_dataframe_agent(llm, df, verbose=True, allow_dangerous_code=True)
+
+# Wrap it as a tool
+@function_tool
+def df_agent(query: str):
+    """Query the healthcare dataset using natural language."""
+    response = pandas_agent.invoke(query)
+    return response
+
+
+
+
+DataAnalyst_agent = Agent(
+    name="DataAnalyst_agent",
+    instructions=(
+        "You are a data analyst and you are given a dataframe tool. ""You need to analyse it and give some insights on it. "
+"You will use the dataframe tool to analyse, give the report, and visualizations also."
+    ),
+    model="gpt-4o-mini",
+    tools=[df_agent],   
+) 
+Business_Intelligence_Agent = Agent(
+    name="Business Intelligence Agent",
+    instructions="You are a business intelligence agent and you are given a dataframe tool and you need to analyse it and give some insights on it",
+    model="gpt-4o-mini",
+    tools=[df_agent],
+)
+
+Data_Scientist_Agent = Agent(
+    name="Data Scientist Agent",
+    instructions="You are a data scientist and you are given a dataframe tool and you need to analyse it and give some insights on it",
+    model="gpt-4o-mini",
+    tools=[df_agent],
+)
+
+
+
+
+#Rephraser nlp agent 
+rephraser_agent = Agent(
+    name="rephraser_agent",
+    instructions="You rephrase the user's message to make it more natural and fluent",
+    handoff_description="An english to rephraser",
+    model="gpt-4o-mini"
+)
+
+# --- Define Orchestrator Agent ---
+orchestrator_agent = Agent(
+    name="orchestrator_agent",
+    instructions=(
+        "You are a data analysis orchestrator. You receive a user's query and use the df_agent "
+        "tool to analyze the healthcare dataset and provide an insightful response."
+        "You never analyze the data on your own — always use df_agent."
+    ),
+    tools=[
+        DataAnalyst_agent.as_tool(
+            tool_name="analyze_dataframe",
+            tool_description="Analyze the healthcare dataset based on user queries."
+        ),
+         rephraser_agent.as_tool(
+            tool_name="rephrase",
+            tool_description="Rephrase the user's message to make it more natural and fluent",
+        ),
+        Business_Intelligence_Agent.as_tool(
+            tool_name="Business_Intelligence_Agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+        Data_Scientist_Agent.as_tool(
+            tool_name="DataScientist_Agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+    ],
+)
+
+
+# --- Define Synthesizer Agent ---
+synthesizer_agent = Agent(
+    name="synthesizer_agent",
+    instructions=(
+        "You review the dataframe analysis results, clean up the language if needed, "
+        "and present a clear final answer to the user."
+    ),
+)
+
+
+# # --- Main Async Runner ---
+# #without_streamming 
+async def main(query):
+    #   
+
+    # Run orchestrator
+    orchestrator_result = await Runner.run(orchestrator_agent, query)
+
+    for item in orchestrator_result.new_items:
+        print(f"Intermediate result: {item}")
+
+    # Run synthesizer
+    synthesizer_result = await Runner.run(
+        synthesizer_agent, orchestrator_result.to_input_list()
+    )
+
+    print(f"\n\nFinal Answer:\n{synthesizer_result.final_output}")
+
+
+#with_streamming
+# --- Main Async Function with Streaming + Trace ---
+# async def main(query: str):
+#     with trace("Dataframe analysis trace"):
+#         print("Analyzing your query...\n")
+
+#         # Streamed orchestration
+#         result = Runner.run_streamed(orchestrator_agent, query)
+
+#         async for event in result.stream_events():
+#             if (
+#                 event.type == "run_item_stream_event"
+#                 and event.item.type == "tool_call_item"
+#             ):
+#                 print(f"\n🔧 Tool Call: {event.item.name}")
+#             elif (
+#                 event.type == "run_item_stream_event"
+#                 and event.item.type == "message_output_item"
+#             ):
+#                 print(f"🧠 Model Output: {event.item.raw_item.output_text}")
+#             elif event.type == "run_step_stream_event":
+#                 print(f"Step Event: {event.step_name}")
+#             else:
+#                 print(f"Other Event: {event.type}")
+
+#         print("\nSynthesizing final output...\n")
+
+#         # Run the synthesizer for final cleanup
+#         synthesizer_result = await Runner.run(
+#             synthesizer_agent, result.to_input_list()
+#         )
+
+#         print(f"\n✅ Final Answer:\n{synthesizer_result.final_output}")
+
+if __name__ == "__main__":
+    user_query = input("Enter the query: ")
+    asyncio.run(main(user_query))

Routes/main_agent_chat_bot_v2.py

@@ -0,0 +1,178 @@
+
+
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional, List, Dict
+from datetime import datetime
+from Redis.sessions_new import *
+from .helpers.main_agent_best_as_of_now import main 
+
+
+main_chatbot_route_v2 = APIRouter(prefix="/main_chatbot_v2", tags=["main_chatbot_v2"])
+
+# ==================== UPDATED PYDANTIC MODELS ====================
+
+class Author(BaseModel):
+    role: str
+
+class ChatRequest(BaseModel):
+    user_login_id: str
+    session_id: Optional[str] = None
+    query: str
+    org_id: Optional[str] = None
+    metadata: Optional[Dict] = None
+    list_of_files_path: Optional[List[str]] = None
+    author: Optional[Author] = None
+    create_time: Optional[str] = None
+    token: Optional[str] = None
+
+class ChatResponse(BaseModel):
+    session_id: str
+    user_message: str
+    assistant_response: str
+    is_new_session: bool
+    session_title: str
+    timestamp: str
+    files_processed: Optional[List[str]] = None
+
+# ==================== ENDPOINTS ====================
+
+@main_chatbot_route_v2.get("/")
+async def root():
+    return {
+        "message": "AI Data Analysis Chatbot API is running!",
+        "version": "2.0.0",
+        "features": ["Session Management", "Chat History", "Context Awareness", "Auto Title", "File Processing"]
+    }
+
+@main_chatbot_route_v2.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest):
+    """
+    Main chat endpoint with automatic session management and file processing
+    
+    Args:
+        user_login_id: User identifier
+        session_id: Session ID (None for new chat)
+        query: User's message
+        org_id: Optional organization ID
+        metadata: Optional metadata (source, language, etc.)
+        list_of_files_path: Optional list of file paths to process
+        author: Optional author information with role
+        create_time: Optional timestamp of request creation
+        token: Optional authentication token
+    
+    Returns:
+        ChatResponse with session details and AI response
+    """
+    try:
+        is_new_session = False
+        
+        # ==================== STEP 1: Session Detection ====================
+        if request.session_id is None:
+            # NEW CHAT → Create new session
+            session_data = create_session(
+                user_login_id=request.user_login_id,
+                org_id=request.org_id,
+                metadata=request.metadata
+            )
+            session_id = session_data["session_id"]
+            is_new_session = True
+            conversation_history = []
+        else:
+            # EXISTING CHAT → Load session and history
+            try:
+                session_data = get_session(request.user_login_id, request.session_id)
+                session_id = request.session_id
+                conversation_history = get_message_history(
+                    request.user_login_id, 
+                    session_id, 
+                    limit=5
+                )
+            except HTTPException as e:
+                if e.status_code == 404:
+                    # Session expired → Create new one
+                    session_data = create_session(
+                        user_login_id=request.user_login_id,
+                        org_id=request.org_id,
+                        metadata=request.metadata
+                    )
+                    session_id = session_data["session_id"]
+                    is_new_session = True
+                    conversation_history = []
+                else:
+                    raise
+
+        # ==================== STEP 2: Save User Message ====================
+        # Include files info in the message if provided
+        user_message_content = request.query
+        if request.list_of_files_path:
+            user_message_content += f"\n[Files: {', '.join(request.list_of_files_path)}]"
+        
+        add_message(
+            user_login_id=request.user_login_id,
+            session_id=session_id,
+            role="user",
+            content=user_message_content
+        )
+
+        # ==================== STEP 3: Process with AI Agent ====================
+        # Format context from conversation history
+        context = format_conversation_context(conversation_history) if conversation_history else ""
+        
+        # Prepare additional context for the agent
+        agent_context = {
+            "query": request.query,
+            "files": request.list_of_files_path,
+            "conversation_history": context,
+            "org_id": request.org_id,
+            "metadata": request.metadata,
+            "author_role": request.author.role if request.author else None
+        }
+        
+        # Call AI agent - modify based on your main() function signature
+        # Option 1: If main() accepts multiple parameters
+        # result = await main(request.query, files=request.list_of_files_path, context=context)
+        
+        # Option 2: If main() accepts a single query parameter (current)
+        print("File paths received in request:", request.list_of_files_path)
+        result = await main(request.query,file_paths=request.list_of_files_path)
+        print("Result from main():", result)
+        
+        # Option 3: If you want to pass everything
+        # result = await main(agent_context)
+
+        # ==================== STEP 4: Save Assistant Response ====================
+        add_message(
+            user_login_id=request.user_login_id,
+            session_id=session_id,
+            role="assistant",
+            content=result
+        )
+
+        # ==================== STEP 5: Auto-Generate Title ====================
+        update_session_title_if_needed(
+            request.user_login_id, 
+            session_id, 
+            is_new_session
+        )
+        
+        # Get updated session with title
+        session_data = get_session(request.user_login_id, session_id)
+
+        # ==================== STEP 6: Return Response ====================
+        return ChatResponse(
+            session_id=session_id,
+            user_message=request.query,
+            assistant_response=result,
+            is_new_session=is_new_session,
+            session_title=session_data.get("title", "New Chat"),
+            timestamp=datetime.now().isoformat(),
+            files_processed=request.list_of_files_path,
+            artifacts_paths=[]
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error in chat: {str(e)}")

Routes/main_chat_bot.py

@@ -0,0 +1,588 @@
+# from fastapi import APIRouter, HTTPException
+# from pydantic import BaseModel
+# from typing import Optional, List, Dict
+# import json
+# import redis
+# import os
+# import uuid
+# from datetime import datetime
+# from .helpers.main_agent_helpers import main
+# from dotenv import load_dotenv
+
+# load_dotenv()
+
+# # Initialize router
+# main_chatbot_route = APIRouter(prefix="/main_chatbot", tags=["main_chatbot"])
+
+# # ==================== REDIS CLIENT ====================
+
+# def get_redis_client():
+#     """Initialize Redis client"""
+#     try:
+#         REDIS_URL = os.getenv("REDIS_URL")
+#         REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
+#         REDIS_PORT = int(os.getenv("REDIS_PORT", 6379))
+#         REDIS_PASSWORD = os.getenv("REDIS_PASSWORD")
+        
+#         if REDIS_URL:
+#             redis_client = redis.from_url(
+#                 REDIS_URL,
+#                 decode_responses=True,
+#                 socket_connect_timeout=5,
+#                 socket_timeout=5
+#             )
+#         else:
+#             redis_client = redis.StrictRedis(
+#                 host=REDIS_HOST,
+#                 port=REDIS_PORT,
+#                 password=REDIS_PASSWORD,
+#                 decode_responses=True,
+#                 socket_connect_timeout=5,
+#                 socket_timeout=5
+#             )
+        
+#         redis_client.ping()
+#         print(f"✅ Redis connected for chatbot")
+#         return redis_client
+#     except Exception as e:
+#         print(f"❌ Redis connection failed: {e}")
+#         raise HTTPException(status_code=500, detail=f"Redis connection failed: {str(e)}")
+
+# redis_client = get_redis_client()
+
+# # ==================== PYDANTIC MODELS ====================
+
+# class ChatRequest(BaseModel):
+#     user_login_id: str
+#     session_id: Optional[str] = None  # None = NEW CHAT
+#     query: str
+#     org_id: Optional[str] = None
+#     metadata: Optional[Dict] = None
+
+# class ChatResponse(BaseModel):
+#     session_id: str
+#     user_message: str
+#     assistant_response: str
+#     is_new_session: bool
+#     session_title: str
+#     timestamp: str
+
+# class MessageResponse(BaseModel):
+#     message_id: str
+#     role: str
+#     content: str
+#     timestamp: str
+
+# class ChatHistoryResponse(BaseModel):
+#     session_id: str
+#     title: str
+#     created_at: str
+#     message_count: int
+#     messages: List[MessageResponse]
+
+# # ==================== SESSION MANAGEMENT FUNCTIONS ====================
+
+# def create_session(user_login_id: str, org_id: Optional[str] = None, metadata: Optional[Dict] = None) -> dict:
+#     """Create a new chat session"""
+#     session_id = str(uuid.uuid4())
+#     session_data = {
+#         "session_id": session_id,
+#         "user_login_id": user_login_id,
+#         "org_id": org_id,
+#         "created_at": datetime.now().isoformat(),
+#         "title": "New Chat",
+#         "message_count": 0,
+#         "metadata": metadata or {}
+#     }
+    
+#     # Store session in Redis with 24 hour TTL
+#     # Key pattern: session:{user_login_id}:{session_id}
+#     redis_client.setex(
+#         f"session:{user_login_id}:{session_id}",
+#         86400,  # 24 hours
+#         json.dumps(session_data)
+#     )
+    
+#     # Initialize empty message history
+#     # Key pattern: messages:{user_login_id}:{session_id}
+#     redis_client.setex(
+#         f"messages:{user_login_id}:{session_id}",
+#         86400,  # 24 hours
+#         json.dumps([])
+#     )
+    
+#     return session_data
+
+# def get_session(user_login_id: str, session_id: str) -> dict:
+#     """Get session data from Redis"""
+#     session_key = f"session:{user_login_id}:{session_id}"
+#     session_data = redis_client.get(session_key)
+    
+#     if not session_data:
+#         raise HTTPException(status_code=404, detail="Session not found or expired")
+    
+#     return json.loads(session_data)
+
+# def add_message(user_login_id: str, session_id: str, role: str, content: str) -> str:
+#     """Add message to session chat history"""
+#     message_id = str(uuid.uuid4())
+#     message_data = {
+#         "message_id": message_id,
+#         "role": role,  # "user" or "assistant"
+#         "content": content,
+#         "timestamp": datetime.now().isoformat()
+#     }
+    
+#     # Get current message history
+#     messages_key = f"messages:{user_login_id}:{session_id}"
+#     messages_data = redis_client.get(messages_key)
+    
+#     if messages_data:
+#         messages = json.loads(messages_data)
+#     else:
+#         messages = []
+    
+#     # Add new message
+#     messages.append(message_data)
+    
+#     # Update messages in Redis
+#     redis_client.setex(messages_key, 86400, json.dumps(messages))
+    
+#     # Update session message count
+#     session_key = f"session:{user_login_id}:{session_id}"
+#     session_data = redis_client.get(session_key)
+#     if session_data:
+#         session = json.loads(session_data)
+#         session["message_count"] = len(messages)
+#         redis_client.setex(session_key, 86400, json.dumps(session))
+    
+#     return message_id
+
+# def get_message_history(user_login_id: str, session_id: str, limit: Optional[int] = None) -> List[Dict]:
+#     """Get message history for a session"""
+#     messages_key = f"messages:{user_login_id}:{session_id}"
+#     messages_data = redis_client.get(messages_key)
+    
+#     if not messages_data:
+#         return []
+    
+#     messages = json.loads(messages_data)
+    
+#     # Return last 'limit' messages if specified
+#     if limit:
+#         return messages[-limit:] if len(messages) > limit else messages
+    
+#     return messages
+
+# def generate_session_title(user_login_id: str, session_id: str) -> str:
+#     """Generate a title for the session based on first user message"""
+#     try:
+#         # Get message history
+#         messages = get_message_history(user_login_id, session_id)
+        
+#         if not messages:
+#             return "New Chat"
+        
+#         # Get first user message
+#         first_user_message = next(
+#             (msg["content"] for msg in messages if msg["role"] == "user"), 
+#             None
+#         )
+        
+#         if not first_user_message:
+#             return "New Chat"
+        
+#         # Create simple title from first message (first 6 words)
+#         words = first_user_message.split()[:6]
+#         title = " ".join(words) + ("..." if len(first_user_message.split()) > 6 else "")
+        
+#         # Update session with new title
+#         session_key = f"session:{user_login_id}:{session_id}"
+#         session_data = redis_client.get(session_key)
+        
+#         if session_data:
+#             session = json.loads(session_data)
+#             session["title"] = title
+#             redis_client.setex(session_key, 86400, json.dumps(session))
+        
+#         return title
+        
+#     except Exception as e:
+#         print(f"Error generating session title: {e}")
+#         return "New Chat"
+
+# def update_session_title_if_needed(user_login_id: str, session_id: str, is_new_session: bool):
+#     """Update session title after first message"""
+#     try:
+#         session = get_session(user_login_id, session_id)
+        
+#         # Only update if it's a new session or title is still "New Chat"
+#         if is_new_session or session.get("title") == "New Chat":
+#             generate_session_title(user_login_id, session_id)
+            
+#     except Exception as e:
+#         print(f"Error updating session title: {e}")
+
+# def format_conversation_context(messages: List[Dict], max_messages: int = 5) -> str:
+#     """Format conversation history as context for the AI agent"""
+#     if not messages:
+#         return ""
+    
+#     # Get last N messages for context
+#     recent_messages = messages[-max_messages:] if len(messages) > max_messages else messages
+    
+#     context_lines = []
+#     for msg in recent_messages:
+#         context_lines.append(f"{msg['role']}: {msg['content']}")
+    
+#     return "\n".join(context_lines)
+
+# # ==================== ENDPOINTS ====================
+
+# @main_chatbot_route.get("/")
+# async def root():
+#     """Root endpoint"""
+#     return {
+#         "message": "AI Data Analysis Chatbot API is running!",
+#         "version": "2.0.0",
+#         "features": [
+#             "Session Management",
+#             "Auto New Chat Detection",
+#             "Chat History",
+#             "Context Awareness",
+#             "Auto Title Generation"
+#         ]
+#     }
+
+# @main_chatbot_route.post("/chat", response_model=ChatResponse)
+# async def chat(request: ChatRequest):
+#     """
+#     Main chat endpoint with automatic session management
+    
+#     Flow:
+#     1. If session_id is None → Create new session
+#     2. If session_id exists → Load existing session
+#     3. Save user message
+#     4. Process with AI agent (with conversation context)
+#     5. Save assistant response
+#     6. Auto-generate title if new session
+#     7. Return response
+    
+#     Args:
+#         user_login_id: User identifier
+#         session_id: Session ID (None for new chat)
+#         query: User's message
+#         org_id: Optional organization ID
+#         metadata: Optional metadata
+    
+#     Returns:
+#         ChatResponse with session details and AI response
+#     """
+#     try:
+#         is_new_session = False
+#         session_data = None
+        
+#         # ==================== STEP 1: Session Detection ====================
+#         if request.session_id is None:
+#             # NEW CHAT → Create new session
+#             session_data = create_session(
+#                 user_login_id=request.user_login_id,
+#                 org_id=request.org_id,
+#                 metadata=request.metadata
+#             )
+#             session_id = session_data["session_id"]
+#             is_new_session = True
+#             conversation_history = []
+            
+#         else:
+#             # EXISTING CHAT → Load session and history
+#             try:
+#                 session_data = get_session(request.user_login_id, request.session_id)
+#                 session_id = request.session_id
+                
+#                 # Get conversation history for context
+#                 conversation_history = get_message_history(
+#                     request.user_login_id, 
+#                     session_id,
+#                     limit=5  # Last 5 messages for context
+#                 )
+                
+#             except HTTPException as e:
+#                 if e.status_code == 404:
+#                     # Session expired → Create new one
+#                     session_data = create_session(
+#                         user_login_id=request.user_login_id,
+#                         org_id=request.org_id,
+#                         metadata=request.metadata
+#                     )
+#                     session_id = session_data["session_id"]
+#                     is_new_session = True
+#                     conversation_history = []
+#                 else:
+#                     raise
+        
+#         # ==================== STEP 2: Save User Message ====================
+#         user_message_id = add_message(
+#             user_login_id=request.user_login_id,
+#             session_id=session_id,
+#             role="user",
+#             content=request.query
+#         )
+        
+#         # ==================== STEP 3: Process with AI Agent ====================
+#         # Format context from conversation history
+#         context = format_conversation_context(conversation_history) if conversation_history else ""
+        
+#         # Call your AI agent
+#         # If your main() function accepts context, modify this:
+#         # result = await main(request.query, context=context)
+#         result = await main(request.query)
+        
+#         assistant_response = str(result)
+        
+#         # ==================== STEP 4: Save Assistant Response ====================
+#         assistant_message_id = add_message(
+#             user_login_id=request.user_login_id,
+#             session_id=session_id,
+#             role="assistant",
+#             content=assistant_response
+#         )
+        
+#         # ==================== STEP 5: Auto-Generate Title ====================
+#         update_session_title_if_needed(
+#             request.user_login_id, 
+#             session_id, 
+#             is_new_session
+#         )
+        
+#         # Get updated session with title
+#         session_data = get_session(request.user_login_id, session_id)
+        
+#         # ==================== STEP 6: Return Response ====================
+#         return ChatResponse(
+#             session_id=session_id,
+#             user_message=request.query,
+#             assistant_response=assistant_response,
+#             is_new_session=is_new_session,
+#             session_title=session_data.get("title", "New Chat"),
+#             timestamp=datetime.now().isoformat()
+#         )
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error in chat: {str(e)}")
+
+# @main_chatbot_route.get("/history/{user_login_id}/{session_id}", response_model=ChatHistoryResponse)
+# async def get_chat_history(user_login_id: str, session_id: str):
+#     """
+#     Get complete chat history for a session
+    
+#     Args:
+#         user_login_id: User identifier
+#         session_id: Session ID
+    
+#     Returns:
+#         ChatHistoryResponse with all messages
+#     """
+#     try:
+#         # Get session data
+#         session_data = get_session(user_login_id, session_id)
+        
+#         # Get all messages
+#         messages = get_message_history(user_login_id, session_id)
+        
+#         # Convert to MessageResponse objects
+#         message_responses = [
+#             MessageResponse(
+#                 message_id=msg["message_id"],
+#                 role=msg["role"],
+#                 content=msg["content"],
+#                 timestamp=msg["timestamp"]
+#             )
+#             for msg in messages
+#         ]
+        
+#         return ChatHistoryResponse(
+#             session_id=session_id,
+#             title=session_data.get("title", "New Chat"),
+#             created_at=session_data.get("created_at"),
+#             message_count=len(messages),
+#             messages=message_responses
+#         )
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error fetching history: {str(e)}")
+
+# @main_chatbot_route.get("/sessions/{user_login_id}")
+# async def get_user_sessions(user_login_id: str):
+#     """
+#     Get all sessions for a user
+    
+#     Args:
+#         user_login_id: User identifier
+    
+#     Returns:
+#         List of all sessions for the user
+#     """
+#     try:
+#         sessions = []
+        
+#         # Scan for all session keys for this user
+#         pattern = f"session:{user_login_id}:*"
+#         for key in redis_client.scan_iter(match=pattern):
+#             session_data = redis_client.get(key)
+#             if session_data:
+#                 session = json.loads(session_data)
+#                 sessions.append(session)
+        
+#         # Sort by created_at (most recent first)
+#         sessions.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+        
+#         return {
+#             "user_login_id": user_login_id,
+#             "total_sessions": len(sessions),
+#             "sessions": sessions
+#         }
+        
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error fetching sessions: {str(e)}")
+
+# @main_chatbot_route.delete("/sessions/{user_login_id}/{session_id}")
+# async def delete_session(user_login_id: str, session_id: str):
+#     """
+#     Delete a session and its messages
+    
+#     Args:
+#         user_login_id: User identifier
+#         session_id: Session ID to delete
+    
+#     Returns:
+#         Deletion confirmation
+#     """
+#     try:
+#         # Verify session exists
+#         get_session(user_login_id, session_id)
+        
+#         # Delete session and messages
+#         session_key = f"session:{user_login_id}:{session_id}"
+#         messages_key = f"messages:{user_login_id}:{session_id}"
+        
+#         redis_client.delete(session_key)
+#         redis_client.delete(messages_key)
+        
+#         return {
+#             "message": f"Session {session_id} deleted successfully",
+#             "session_id": session_id,
+#             "user_login_id": user_login_id
+#         }
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error deleting session: {str(e)}")
+
+# @main_chatbot_route.get("/health")
+# async def health():
+#     """Health check endpoint"""
+#     try:
+#         redis_client.ping()
+#         redis_status = "connected"
+        
+#         # Count total sessions
+#         total_sessions = len(list(redis_client.scan_iter(match="session:*")))
+        
+#     except:
+#         redis_status = "disconnected"
+#         total_sessions = 0
+    
+#     return {
+#         "status": "ok",
+#         "service": "Chatbot API with Session Management",
+#         "redis_status": redis_status,
+#         "total_sessions": total_sessions,
+#         "session_ttl": "24 hours"
+#     }
+
+# #============putsession_title===================
+# @main_chatbot_route.put("/sessions/{user_login_id}/{session_id}/title")
+# async def update_session_title(user_login_id: str, session_id: str, title: str):    
+#     try:
+#         session_key = f"session:{user_login_id}:{session_id}"
+#         session_data = redis_client.get(session_key)
+        
+#         if session_data:
+#             session = json.loads(session_data)
+#             session["title"] = title
+#             redis_client.setex(session_key, 86400, json.dumps(session))
+#             return {"message": "Session title updated successfully."}
+#         else:
+#             raise HTTPException(status_code=404, detail="Session not found.")
+        
+#     except HTTPException:
+#         raise
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=f"Error updating session title: {str(e)}")  
+
+#======================main_chatbot=====================
+from fastapi import APIRouter, HTTPException
+from datetime import datetime
+from Redis.sessions_new import *
+from .helpers.main_agent_helpers import main
+
+main_chatbot_route = APIRouter(prefix="/main_chatbot", tags=["main_chatbot"])
+
+@main_chatbot_route.get("/")
+async def root():
+    return {
+        "message": "AI Data Analysis Chatbot API is running!",
+        "version": "2.0.0",
+        "features": ["Session Management", "Chat History", "Context Awareness", "Auto Title"]
+    }
+
+@main_chatbot_route.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest):
+    try:
+        is_new_session = False
+
+        if request.session_id is None:
+            session_data = create_session(request.user_login_id, request.org_id, request.metadata)
+            session_id = session_data["session_id"]
+            is_new_session = True
+            conversation_history = []
+        else:
+            try:
+                session_data = get_session(request.user_login_id, request.session_id)
+                session_id = request.session_id
+                conversation_history = get_message_history(request.user_login_id, session_id, limit=5)
+            except HTTPException as e:
+                if e.status_code == 404:
+                    session_data = create_session(request.user_login_id, request.org_id, request.metadata)
+                    session_id = session_data["session_id"]
+                    is_new_session = True
+                    conversation_history = []
+                else:
+                    raise
+
+        add_message(request.user_login_id, session_id, "user", request.query)
+
+        context = format_conversation_context(conversation_history) if conversation_history else ""
+        result = await main(request.query)  # You can modify to pass context if needed
+
+        add_message(request.user_login_id, session_id, "assistant", result)
+
+        update_session_title_if_needed(request.user_login_id, session_id, is_new_session)
+        session_data = get_session(request.user_login_id, session_id)
+
+        return ChatResponse(
+            session_id=session_id,
+            user_message=request.query,
+            assistant_response=result,
+            is_new_session=is_new_session,
+            session_title=session_data.get("title", "New Chat"),
+            timestamp=datetime.now().isoformat()
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error in chat: {str(e)}")

Routes/main_chat_bot.py_main.txt

@@ -0,0 +1,24 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+from .helpers.main_agent_helpers import main
+from fastapi import APIRouter
+
+main_chatbot_route = APIRouter(prefix="/main_chatbot", tags=["main_chatbot"])
+
+
+# --- Request Model ---
+class ChatRequest(BaseModel):
+    query: str
+
+# --- Endpoints ---
+@main_chatbot_route.get("/")
+async def root():
+    return {"message": "AI Data Analysis API is running!"}
+
+@main_chatbot_route.post("/chat")
+async def chat(request: ChatRequest):
+    try:
+        result = await main(request.query)
+        return {"response": str(result)}  # <-- Wrap in a dictionary and str() to be safe
+    except Exception as e:
+        return {"error": str(e)}

agent_tools/Hybrid_Rag_agent.py

@@ -0,0 +1,93 @@
+import asyncio
+
+from agents import Agent, ItemHelpers, MessageOutputItem, Runner, trace
+
+
+from agents import Agent, Runner, function_tool
+import asyncio
+import os
+import dotenv
+
+
+
+from agents import Agent, ModelSettings, function_tool , Runner
+
+
+
+@function_tool
+def vector_db_agent_tool(query: str) -> str:
+    print("vector_db_agent(query)")
+    
+@function_tool
+def neo4j_agent_tool(query: str) -> str:
+    print("neo4j_agent(query)")
+    
+    
+    
+    
+
+# async def main():
+#     result = await Runner.run(
+#         Dataframe_agent,
+#         "so analayse the df and give some insights on it",
+#     )
+#     print(result)
+# if __name__ == "__main__":
+#    asyncio.run(main())
+
+#custom agents for the project and why 
+
+
+
+
+
+
+orchestrator_agent = Agent(
+    name="orchestrator_agent",
+    instructions=(
+        "You are a translation agent. You use the tools given to you to translate."
+        "If asked for multiple translations, you call the relevant tools in order."
+        "You never translate on your own, you always use the provided tools."
+    ),
+    tools=[
+        
+        vector_db_agent_tool.as_tool(
+            tool_name="vector_db_agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+        neo4j_agent_tool.as_tool(
+            tool_name="neo4j_agent",
+            tool_description="Analyse the dataframe and give some insights on it and will use for reports and visualizations",
+        ),
+
+    ],
+)
+
+synthesizer_agent = Agent(
+    name="synthesizer_agent",
+    instructions="You inspect translations, correct them if needed, and produce a final concatenated response.",
+)
+
+
+async def main():
+    msg = input("Hi! What would you like translated, and to which languages? ")
+
+    # Run the entire orchestration in a single trace
+    with trace("Orchestrator evaluator"):
+        orchestrator_result = await Runner.run(orchestrator_agent, msg)
+
+        for item in orchestrator_result.new_items:
+            if isinstance(item, MessageOutputItem):
+                text = ItemHelpers.text_message_output(item)
+                if text:
+                    print(f"  - Translation step: {text}")
+
+        synthesizer_result = await Runner.run(
+            synthesizer_agent, orchestrator_result.to_input_list()
+        )
+
+    print(f"\n\nFinal response:\n{synthesizer_result.final_output}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

agent_tools/Reflection_agent.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Literal
+import os 
+import dotenv
+dotenv.load_dotenv()
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+from agents import Agent, ItemHelpers, Runner, TResponseInputItem, trace
+
+"""
+This example shows the LLM as a judge pattern. The first agent generates an outline for a story.
+The second agent judges the outline and provides feedback. We loop until the judge is satisfied
+with the outline.
+"""
+
+story_outline_generator = Agent(
+    name="story_outline_generator",
+    instructions=(
+        "You generate a very short story outline based on the user's input. "
+        "If there is any feedback provided, use it to improve the outline."
+    ),
+)
+
+
+@dataclass
+class EvaluationFeedback:
+    feedback: str
+    score: Literal["pass", "needs_improvement", "fail"]
+
+
+evaluator = Agent[None](
+    name="evaluator",
+    instructions=(
+        "You evaluate a story outline and decide if it's good enough. "
+        "If it's not good enough, you provide feedback on what needs to be improved. "
+        "Never give it a pass on the first try. After 5 attempts, you can give it a pass if the story outline is good enough - do not go for perfection"
+    ),
+    output_type=EvaluationFeedback,
+)
+
+
+async def main() -> None:
+    msg = input("What kind of story would you like to hear? ")
+    input_items: list[TResponseInputItem] = [{"content": msg, "role": "user"}]
+
+    latest_outline: str | None = None
+
+    # We'll run the entire workflow in a single trace
+    with trace("LLM as a judge"):
+        while True:
+            story_outline_result = await Runner.run(
+                story_outline_generator,
+                input_items,
+            )
+
+            input_items = story_outline_result.to_input_list()
+            latest_outline = ItemHelpers.text_message_outputs(story_outline_result.new_items)
+            print("Story outline generated")
+
+            evaluator_result = await Runner.run(evaluator, input_items)
+            result: EvaluationFeedback = evaluator_result.final_output
+
+            print(f"Evaluator score: {result.score}")
+
+            if result.score == "pass":
+                print("Story outline is good enough, exiting.")
+                break
+
+            print("Re-running with feedback")
+
+            input_items.append({"content": f"Feedback: {result.feedback}", "role": "user"})
+
+    print(f"Final story outline: {latest_outline}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

agent_tools/autovis_tool.py

@@ -0,0 +1,124 @@
+
+# from autoviz import AutoViz_Class
+# import os
+# import pandas as pd
+# import uuid
+
+# def run_autoviz(
+#     filename,
+#     sep=",",
+#     depVar="",
+#     dfte=None,
+#     header=0,
+#     verbose=2,
+#     lowess=False,
+#     chart_format="html",
+#     max_rows_analyzed=150000,
+#     max_cols_analyzed=30,
+#     save_plot_dir=None
+# ):
+#     # Generate unique directory using uuid
+#     vis_in = str(uuid.uuid4())
+#     save_plot_dir = f"./{chart_format}_{vis_in}"
+#     os.makedirs(save_plot_dir, exist_ok=True)
+
+#     # Read the CSV file to check if the path is correct
+#     df = pd.read_csv(filename)
+#     # print(df.head())
+
+#     # Run AutoViz
+#     AV = AutoViz_Class()
+#     print("Running AutoViz...")
+#     dft = AV.AutoViz(
+#         filename,
+#         sep=sep,
+#         depVar=depVar,
+#         dfte=dfte,
+#         header=header,
+#         verbose=verbose,
+#         lowess=lowess,
+#         chart_format=chart_format,
+#         max_rows_analyzed=max_rows_analyzed,
+#         max_cols_analyzed=max_cols_analyzed,
+#         save_plot_dir=save_plot_dir
+#     )
+#     print(dft)
+#     print(f"Plots saved in: {save_plot_dir}")
+#     return dft
+
+# # Example usage:
+# run_autoviz(
+#     filename=r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+# )
+
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+from autoviz import AutoViz_Class
+import os
+import pandas as pd
+import uuid
+from fastapi import APIRouter
+
+Autoviz_router = APIRouter(prefix="/autoviz", tags=["autoviz"])
+
+
+class AutoVizParams(BaseModel):
+    filename: str
+    sep: str = ","
+    depVar: str = ""
+    header: int = 0
+    verbose: int = 2
+    lowess: bool = False
+    chart_format: str = "html"
+    max_rows_analyzed: int = 150000
+    max_cols_analyzed: int = 30
+
+def run_autoviz(
+    filename,
+    sep=",",
+    depVar="",
+    dfte=None,
+    header=0,
+    verbose=2,
+    lowess=False,
+    chart_format="html",
+    max_rows_analyzed=150000,
+    max_cols_analyzed=30,
+    save_plot_dir=None
+):
+    vis_in = str(uuid.uuid4())
+    save_plot_dir = f"./{chart_format}_{vis_in}"
+    os.makedirs(save_plot_dir, exist_ok=True)
+    df = pd.read_csv(filename)
+    AV = AutoViz_Class()
+    dft = AV.AutoViz(
+        filename,
+        sep=sep,
+        depVar=depVar,
+        dfte=dfte,
+        header=header,
+        verbose=verbose,
+        lowess=lowess,
+        chart_format=chart_format,
+        max_rows_analyzed=max_rows_analyzed,
+        max_cols_analyzed=max_cols_analyzed,
+        save_plot_dir=save_plot_dir
+    )
+    return {"message": "AutoViz run complete", "plots_dir": save_plot_dir}
+
+@Autoviz_router.post("/run_autoviz")
+async def autoviz_api(params: AutoVizParams):
+    result = run_autoviz(
+        filename=params.filename,
+        sep=params.sep,
+        depVar=params.depVar,
+        dfte=None,
+        header=params.header,
+        verbose=params.verbose,
+        lowess=params.lowess,
+        chart_format=params.chart_format,
+        max_rows_analyzed=params.max_rows_analyzed,
+        max_cols_analyzed=params.max_cols_analyzed,
+        save_plot_dir=None
+    )
+    return result

agent_tools/code_intepreter.py

@@ -0,0 +1,67 @@
+import asyncio
+import os
+import dotenv
+import pandas as pd
+
+from agents import Agent, CodeInterpreterTool, Runner, trace
+
+dotenv.load_dotenv()
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
+
+CSV_PATH = r"C:\Users\Dell\Documents\MR-AI\openai_agents\healthcare-data-30.csv"
+
+async def main():
+    if not os.path.exists(CSV_PATH):
+        raise FileNotFoundError(f"❌ CSV file not found at {CSV_PATH}")
+
+    df = pd.read_csv(CSV_PATH)
+    print(f"📄 Loaded CSV data (first 5 rows):\n{df.head()}\n")
+
+    # Prepare textual context for the model
+    df_preview = df.head(10).to_markdown()
+    data_context = (
+        f"The dataset preview is:\n{df_preview}\n\n"
+        f"Column headers: {', '.join(df.columns)}"
+    )
+
+    # Define the agent
+    agent = Agent(
+        name="CSV Code Interpreter",
+        model="gpt-4.1",
+        instructions=(
+            "You are a data analyst who loves working with CSV data. "
+            "You can analyze and visualize data from pandas DataFrames using Python code. "
+            "Use the provided CSV data to answer questions.\n"
+        ),
+        tools=[
+            CodeInterpreterTool(
+                tool_config={"type": "code_interpreter", "container": {"type": "auto"}}
+            )
+        ],
+    )
+
+    query = input("💬 Ask a question about the CSV file: ")
+
+    with trace("CSV code interpreter example"):
+        print("\n🔍 Processing query...")
+
+        # ✅ Combine the CSV context + user query in one go
+        result = Runner.run_streamed(
+            agent,
+            f"Dataset context:\n{data_context}\n\nUser query: {query}"
+        )
+
+        async for event in result.stream_events():
+            if (
+                event.type == "run_item_stream_event"
+                and event.item.type == "tool_call_item"
+                and event.item.raw_item.type == "code_interpreter_call"
+            ):
+                print(f"\n💻 Code executed:\n```\n{event.item.raw_item.code}\n```\n")
+            elif event.type == "run_item_stream_event":
+                print(f"Other event: {event.item.type}")
+
+        print(f"\n✅ Final output: {result.final_output}")
+
+if __name__ == "__main__":
+    asyncio.run(main())

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,94 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+import sys
+import os
+from pathlib import Path
+
+# Add the project root to Python path
+project_root = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(project_root))
+
+from backend.database import Base, DATABASE_URL
+from backend import models  # Import all models to register them
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+target_metadata = Base.metadata
+
+# Set the database URL for Alembic
+config.set_main_option('sqlalchemy.url', DATABASE_URL)
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/049cf12dc407_removed_data_column_and_added_file_.py

@@ -0,0 +1,52 @@
+"""removed data column and added file_metadata column in conversation_data
+
+Revision ID: 049cf12dc407
+Revises: 519b15d0dca6
+Create Date: 2025-10-27 16:27:22.904582
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '049cf12dc407'
+down_revision: Union[str, None] = '519b15d0dca6'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('idx_convo_id_is_saved', table_name='stored_convoId_data')
+    op.drop_table('stored_convoId_data')
+    op.add_column('conversation_data', sa.Column('is_saved', sa.Boolean(), nullable=False))
+    op.add_column('conversation_data', sa.Column('file_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
+    op.add_column('conversation_data', sa.Column('response', postgresql.JSONB(astext_type=sa.Text()), nullable=False))
+    op.add_column('conversation_data', sa.Column('user_query', postgresql.JSONB(astext_type=sa.Text()), nullable=False))
+    op.drop_column('conversation_data', 'data')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('conversation_data', sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False))
+    op.drop_column('conversation_data', 'user_query')
+    op.drop_column('conversation_data', 'response')
+    op.drop_column('conversation_data', 'file_metadata')
+    op.drop_column('conversation_data', 'is_saved')
+    op.create_table('stored_convoId_data',
+    sa.Column('id', sa.INTEGER(), server_default=sa.text('nextval(\'"stored_convoId_data_id_seq"\'::regclass)'), autoincrement=True, nullable=False),
+    sa.Column('convo_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('user_query', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('response', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('file_metadata', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True),
+    sa.Column('is_saved', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='stored_convoId_data_pkey')
+    )
+    op.create_index('idx_convo_id_is_saved', 'stored_convoId_data', ['convo_id', 'is_saved'], unique=False)
+    # ### end Alembic commands ###

alembic/versions/0f27f624c0f9_make_created_by_non_nullable.py

@@ -0,0 +1,30 @@
+"""make created_by non-nullable
+
+Revision ID: 0f27f624c0f9
+Revises: 872b723d49c9
+Create Date: 2025-10-26 05:01:28.791953
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '0f27f624c0f9'
+down_revision: Union[str, None] = '872b723d49c9'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.alter_column('organizations', 'created_by', existing_type=sa.UUID(), nullable=False)
+    op.alter_column('plans', 'created_by', existing_type=sa.UUID(), nullable=False)
+    op.alter_column('third_party_providers', 'created_by', existing_type=sa.UUID(), nullable=False)
+
+
+def downgrade() -> None:
+    op.alter_column('organizations', 'created_by', existing_type=sa.UUID(), nullable=True)
+    op.alter_column('plans', 'created_by', existing_type=sa.UUID(), nullable=True)
+    op.alter_column('third_party_providers', 'created_by', existing_type=sa.UUID(), nullable=True)

alembic/versions/2cb6dd9a9f5b_improve_models_add_indexes_cascades_.py

@@ -0,0 +1,242 @@
+"""Improve models: add indexes, cascades, timezone-aware timestamps, soft delete, and relationships
+
+Revision ID: 2cb6dd9a9f5b
+Revises: c378ad11cd73
+Create Date: 2025-10-24 11:36:37.693119
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '2cb6dd9a9f5b'
+down_revision: Union[str, None] = 'c378ad11cd73'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('conversation_data', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('conversation_data', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.add_column('datasets', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('datasets', 'submitted_time',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.alter_column('datasets', 'status',
+               existing_type=postgresql.ENUM('pending', 'approved', 'rejected', 'published', name='datasetstatus'),
+               nullable=False)
+    op.alter_column('datasets', 'approved_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               existing_nullable=True)
+    op.alter_column('datasets', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.create_index(op.f('ix_datasets_provider_id'), 'datasets', ['provider_id'], unique=False)
+    op.create_index(op.f('ix_datasets_reviewed_by'), 'datasets', ['reviewed_by'], unique=False)
+    op.create_index(op.f('ix_datasets_status'), 'datasets', ['status'], unique=False)
+    op.drop_constraint('datasets_provider_id_fkey', 'datasets', type_='foreignkey')
+    op.drop_constraint('datasets_reviewed_by_fkey', 'datasets', type_='foreignkey')
+    op.create_foreign_key(None, 'datasets', 'users', ['reviewed_by'], ['id'], ondelete='SET NULL')
+    op.create_foreign_key(None, 'datasets', 'third_party_providers', ['provider_id'], ['id'], ondelete='RESTRICT')
+    op.drop_column('datasets', 'is_approved')
+    op.add_column('leads', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('leads', 'status',
+               existing_type=postgresql.ENUM('new', 'contacted', 'closed', name='leadstatus'),
+               nullable=False)
+    op.alter_column('leads', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.alter_column('leads', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.create_index(op.f('ix_leads_dataset_id'), 'leads', ['dataset_id'], unique=False)
+    op.create_index(op.f('ix_leads_organization_id'), 'leads', ['organization_id'], unique=False)
+    op.create_index(op.f('ix_leads_provider_id'), 'leads', ['provider_id'], unique=False)
+    op.create_index(op.f('ix_leads_status'), 'leads', ['status'], unique=False)
+    op.drop_constraint('leads_organization_id_fkey', 'leads', type_='foreignkey')
+    op.drop_constraint('leads_provider_id_fkey', 'leads', type_='foreignkey')
+    op.drop_constraint('leads_dataset_id_fkey', 'leads', type_='foreignkey')
+    op.create_foreign_key(None, 'leads', 'datasets', ['dataset_id'], ['id'], ondelete='RESTRICT')
+    op.create_foreign_key(None, 'leads', 'organizations', ['organization_id'], ['id'], ondelete='CASCADE')
+    op.create_foreign_key(None, 'leads', 'third_party_providers', ['provider_id'], ['id'], ondelete='RESTRICT')
+    op.add_column('organizations', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('organizations', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=False)
+    op.alter_column('organizations', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.alter_column('organizations', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.create_index(op.f('ix_organizations_plan_id'), 'organizations', ['plan_id'], unique=False)
+    op.drop_constraint('organizations_plan_id_fkey', 'organizations', type_='foreignkey')
+    op.create_foreign_key(None, 'organizations', 'plans', ['plan_id'], ['id'], ondelete='RESTRICT')
+    op.add_column('plans', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('plans', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.add_column('third_party_providers', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('third_party_providers', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=False)
+    op.alter_column('third_party_providers', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.add_column('users', sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True))
+    op.alter_column('users', 'password',
+               existing_type=sa.VARCHAR(length=255),
+               type_=sa.String(length=500),
+               existing_nullable=False)
+    op.alter_column('users', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=False)
+    op.alter_column('users', 'last_login',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               existing_nullable=True)
+    op.alter_column('users', 'created_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.alter_column('users', 'updated_at',
+               existing_type=postgresql.TIMESTAMP(),
+               type_=sa.DateTime(timezone=True),
+               nullable=False)
+    op.create_index(op.f('ix_users_is_active'), 'users', ['is_active'], unique=False)
+    op.create_index(op.f('ix_users_organization_id'), 'users', ['organization_id'], unique=False)
+    op.create_index(op.f('ix_users_provider_id'), 'users', ['provider_id'], unique=False)
+    op.create_index(op.f('ix_users_role'), 'users', ['role'], unique=False)
+    op.drop_constraint('users_provider_id_fkey', 'users', type_='foreignkey')
+    op.drop_constraint('users_organization_id_fkey', 'users', type_='foreignkey')
+    op.create_foreign_key(None, 'users', 'third_party_providers', ['provider_id'], ['id'], ondelete='SET NULL')
+    op.create_foreign_key(None, 'users', 'organizations', ['organization_id'], ['id'], ondelete='SET NULL')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'users', type_='foreignkey')
+    op.drop_constraint(None, 'users', type_='foreignkey')
+    op.create_foreign_key('users_organization_id_fkey', 'users', 'organizations', ['organization_id'], ['id'])
+    op.create_foreign_key('users_provider_id_fkey', 'users', 'third_party_providers', ['provider_id'], ['id'])
+    op.drop_index(op.f('ix_users_role'), table_name='users')
+    op.drop_index(op.f('ix_users_provider_id'), table_name='users')
+    op.drop_index(op.f('ix_users_organization_id'), table_name='users')
+    op.drop_index(op.f('ix_users_is_active'), table_name='users')
+    op.alter_column('users', 'updated_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('users', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('users', 'last_login',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               existing_nullable=True)
+    op.alter_column('users', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=True)
+    op.alter_column('users', 'password',
+               existing_type=sa.String(length=500),
+               type_=sa.VARCHAR(length=255),
+               existing_nullable=False)
+    op.drop_column('users', 'deleted_at')
+    op.alter_column('third_party_providers', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('third_party_providers', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=True)
+    op.drop_column('third_party_providers', 'deleted_at')
+    op.alter_column('plans', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.drop_column('plans', 'deleted_at')
+    op.drop_constraint(None, 'organizations', type_='foreignkey')
+    op.create_foreign_key('organizations_plan_id_fkey', 'organizations', 'plans', ['plan_id'], ['id'])
+    op.drop_index(op.f('ix_organizations_plan_id'), table_name='organizations')
+    op.alter_column('organizations', 'updated_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('organizations', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('organizations', 'is_active',
+               existing_type=sa.BOOLEAN(),
+               nullable=True)
+    op.drop_column('organizations', 'deleted_at')
+    op.drop_constraint(None, 'leads', type_='foreignkey')
+    op.drop_constraint(None, 'leads', type_='foreignkey')
+    op.drop_constraint(None, 'leads', type_='foreignkey')
+    op.create_foreign_key('leads_dataset_id_fkey', 'leads', 'datasets', ['dataset_id'], ['id'])
+    op.create_foreign_key('leads_provider_id_fkey', 'leads', 'third_party_providers', ['provider_id'], ['id'])
+    op.create_foreign_key('leads_organization_id_fkey', 'leads', 'organizations', ['organization_id'], ['id'])
+    op.drop_index(op.f('ix_leads_status'), table_name='leads')
+    op.drop_index(op.f('ix_leads_provider_id'), table_name='leads')
+    op.drop_index(op.f('ix_leads_organization_id'), table_name='leads')
+    op.drop_index(op.f('ix_leads_dataset_id'), table_name='leads')
+    op.alter_column('leads', 'updated_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('leads', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('leads', 'status',
+               existing_type=postgresql.ENUM('new', 'contacted', 'closed', name='leadstatus'),
+               nullable=True)
+    op.drop_column('leads', 'deleted_at')
+    op.add_column('datasets', sa.Column('is_approved', sa.BOOLEAN(), autoincrement=False, nullable=True))
+    op.drop_constraint(None, 'datasets', type_='foreignkey')
+    op.drop_constraint(None, 'datasets', type_='foreignkey')
+    op.create_foreign_key('datasets_reviewed_by_fkey', 'datasets', 'users', ['reviewed_by'], ['id'])
+    op.create_foreign_key('datasets_provider_id_fkey', 'datasets', 'third_party_providers', ['provider_id'], ['id'])
+    op.drop_index(op.f('ix_datasets_status'), table_name='datasets')
+    op.drop_index(op.f('ix_datasets_reviewed_by'), table_name='datasets')
+    op.drop_index(op.f('ix_datasets_provider_id'), table_name='datasets')
+    op.alter_column('datasets', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.alter_column('datasets', 'approved_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               existing_nullable=True)
+    op.alter_column('datasets', 'status',
+               existing_type=postgresql.ENUM('pending', 'approved', 'rejected', 'published', name='datasetstatus'),
+               nullable=True)
+    op.alter_column('datasets', 'submitted_time',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.drop_column('datasets', 'deleted_at')
+    op.alter_column('conversation_data', 'created_at',
+               existing_type=sa.DateTime(timezone=True),
+               type_=postgresql.TIMESTAMP(),
+               nullable=True)
+    op.drop_column('conversation_data', 'deleted_at')
+    # ### end Alembic commands ###

alembic/versions/3b084b14f4b1_add_created_by_field_to_datasets.py

@@ -0,0 +1,34 @@
+"""add created_by field to datasets
+
+Revision ID: 3b084b14f4b1
+Revises: 2cb6dd9a9f5b
+Create Date: 2025-10-24 15:41:23.744370
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '3b084b14f4b1'
+down_revision: Union[str, None] = '2cb6dd9a9f5b'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('datasets', sa.Column('created_by', sa.UUID(), nullable=False))
+    op.create_index(op.f('ix_datasets_created_by'), 'datasets', ['created_by'], unique=False)
+    op.create_foreign_key(None, 'datasets', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'datasets', type_='foreignkey')
+    op.drop_index(op.f('ix_datasets_created_by'), table_name='datasets')
+    op.drop_column('datasets', 'created_by')
+    # ### end Alembic commands ###

alembic/versions/519b15d0dca6_added_two_columns_in_convo_table.py

@@ -0,0 +1,50 @@
+"""added two columns in convo table
+
+Revision ID: 519b15d0dca6
+Revises: 0f27f624c0f9
+Create Date: 2025-10-27 13:07:07.655070
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '519b15d0dca6'
+down_revision: Union[str, None] = '0f27f624c0f9'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('idx_convo_id_is_saved', table_name='stored_convoId_data')
+    op.drop_table('stored_convoId_data')
+    op.add_column('conversation_data', sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False))
+    op.add_column('conversation_data', sa.Column('user_id', sa.UUID(), nullable=False))
+    op.create_index(op.f('ix_conversation_data_user_id'), 'conversation_data', ['user_id'], unique=False)
+    op.create_foreign_key(None, 'conversation_data', 'users', ['user_id'], ['id'], ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'conversation_data', type_='foreignkey')
+    op.drop_index(op.f('ix_conversation_data_user_id'), table_name='conversation_data')
+    op.drop_column('conversation_data', 'user_id')
+    op.drop_column('conversation_data', 'updated_at')
+    op.create_table('stored_convoId_data',
+    sa.Column('id', sa.INTEGER(), server_default=sa.text('nextval(\'"stored_convoId_data_id_seq"\'::regclass)'), autoincrement=True, nullable=False),
+    sa.Column('convo_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('user_query', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('response', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('file_metadata', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True),
+    sa.Column('is_saved', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='stored_convoId_data_pkey')
+    )
+    op.create_index('idx_convo_id_is_saved', 'stored_convoId_data', ['convo_id', 'is_saved'], unique=False)
+    # ### end Alembic commands ###

alembic/versions/872b723d49c9_added_created_by_fields_in_required_.py

@@ -0,0 +1,46 @@
+"""added created_by fields in required tables
+
+Revision ID: 872b723d49c9
+Revises: 3b084b14f4b1
+Create Date: 2025-10-26 04:45:04.299414
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '872b723d49c9'
+down_revision: Union[str, None] = '3b084b14f4b1'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('organizations', sa.Column('created_by', sa.UUID(), nullable=True))
+    op.create_index(op.f('ix_organizations_created_by'), 'organizations', ['created_by'], unique=False)
+    op.create_foreign_key(None, 'organizations', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.add_column('plans', sa.Column('created_by', sa.UUID(), nullable=True))
+    op.create_index(op.f('ix_plans_created_by'), 'plans', ['created_by'], unique=False)
+    op.create_foreign_key(None, 'plans', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.add_column('third_party_providers', sa.Column('created_by', sa.UUID(), nullable=True))
+    op.create_index(op.f('ix_third_party_providers_created_by'), 'third_party_providers', ['created_by'], unique=False)
+    op.create_foreign_key(None, 'third_party_providers', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'third_party_providers', type_='foreignkey')
+    op.drop_index(op.f('ix_third_party_providers_created_by'), table_name='third_party_providers')
+    op.drop_column('third_party_providers', 'created_by')
+    op.drop_constraint(None, 'plans', type_='foreignkey')
+    op.drop_index(op.f('ix_plans_created_by'), table_name='plans')
+    op.drop_column('plans', 'created_by')
+    op.drop_constraint(None, 'organizations', type_='foreignkey')
+    op.drop_index(op.f('ix_organizations_created_by'), table_name='organizations')
+    op.drop_column('organizations', 'created_by')
+    # ### end Alembic commands ###

alembic/versions/ac5b502d055a_added_userdatasetsmetadata_table.py

@@ -0,0 +1,98 @@
+"""Added UserDatasetsMetadata Table
+
+Revision ID: ac5b502d055a
+Revises: 049cf12dc407
+Create Date: 2025-10-29 11:41:42.626176
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = 'ac5b502d055a'
+down_revision: Union[str, None] = '049cf12dc407'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('user_datasets_metadata',
+    sa.Column('id', sa.UUID(), nullable=False),
+    sa.Column('user_id', sa.UUID(), nullable=False),
+    sa.Column('user_metadata', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_user_datasets_metadata_user_id'), 'user_datasets_metadata', ['user_id'], unique=False)
+    op.drop_index('idx_convo_id_is_saved', table_name='stored_convoId_data')
+    op.drop_table('stored_convoId_data')
+    op.drop_index('ix_conversation_data_convo_id', table_name='conversation_data')
+    op.create_index(op.f('ix_conversation_data_convo_id'), 'conversation_data', ['convo_id'], unique=False)
+    op.alter_column('datasets', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=True)
+    op.drop_constraint('datasets_created_by_fkey', 'datasets', type_='foreignkey')
+    op.create_foreign_key(None, 'datasets', 'users', ['created_by'], ['id'], ondelete='SET NULL')
+    op.alter_column('organizations', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=True)
+    op.drop_constraint('organizations_created_by_fkey', 'organizations', type_='foreignkey')
+    op.create_foreign_key(None, 'organizations', 'users', ['created_by'], ['id'], ondelete='SET NULL')
+    op.alter_column('plans', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=True)
+    op.drop_constraint('plans_created_by_fkey', 'plans', type_='foreignkey')
+    op.create_foreign_key(None, 'plans', 'users', ['created_by'], ['id'], ondelete='SET NULL')
+    op.alter_column('third_party_providers', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=True)
+    op.drop_constraint('third_party_providers_created_by_fkey', 'third_party_providers', type_='foreignkey')
+    op.create_foreign_key(None, 'third_party_providers', 'users', ['created_by'], ['id'], ondelete='SET NULL')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'third_party_providers', type_='foreignkey')
+    op.create_foreign_key('third_party_providers_created_by_fkey', 'third_party_providers', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.alter_column('third_party_providers', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=False)
+    op.drop_constraint(None, 'plans', type_='foreignkey')
+    op.create_foreign_key('plans_created_by_fkey', 'plans', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.alter_column('plans', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=False)
+    op.drop_constraint(None, 'organizations', type_='foreignkey')
+    op.create_foreign_key('organizations_created_by_fkey', 'organizations', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.alter_column('organizations', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=False)
+    op.drop_constraint(None, 'datasets', type_='foreignkey')
+    op.create_foreign_key('datasets_created_by_fkey', 'datasets', 'users', ['created_by'], ['id'], ondelete='CASCADE')
+    op.alter_column('datasets', 'created_by',
+               existing_type=sa.UUID(),
+               nullable=False)
+    op.drop_index(op.f('ix_conversation_data_convo_id'), table_name='conversation_data')
+    op.create_index('ix_conversation_data_convo_id', 'conversation_data', ['convo_id'], unique=True)
+    op.create_table('stored_convoId_data',
+    sa.Column('id', sa.INTEGER(), server_default=sa.text('nextval(\'"stored_convoId_data_id_seq"\'::regclass)'), autoincrement=True, nullable=False),
+    sa.Column('convo_id', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('user_query', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('response', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
+    sa.Column('file_metadata', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True),
+    sa.Column('is_saved', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=True),
+    sa.Column('created_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.Column('updated_at', postgresql.TIMESTAMP(), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='stored_convoId_data_pkey')
+    )
+    op.create_index('idx_convo_id_is_saved', 'stored_convoId_data', ['convo_id', 'is_saved'], unique=False)
+    op.drop_index(op.f('ix_user_datasets_metadata_user_id'), table_name='user_datasets_metadata')
+    op.drop_table('user_datasets_metadata')
+    # ### end Alembic commands ###

alembic/versions/c378ad11cd73_initial_migration_capture_current_schema.py

@@ -0,0 +1,30 @@
+"""Initial migration - capture current schema
+
+Revision ID: c378ad11cd73
+Revises: 
+Create Date: 2025-10-24 11:31:53.592361
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'c378ad11cd73'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

app.py

@@ -0,0 +1,91 @@
+import json
+import warnings
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+# Import routes
+# from routes import collections, ingestion, sessions, chat, system
+from Redis.sessions import Redis_session_router
+from Redis.sessions_new import redis_session_route_new
+from Redis.sessions_old import Redis_session_router_old
+from Routes.main_chat_bot import main_chatbot_route
+from Routes.generate_report import Report_Generation_Router
+from s3.file_insertion_s3 import s3_bucket_router
+from backend.main import login_apis_router
+from vector_db.retrival_qa_agent import RetrievalQA_router
+
+
+
+from Routes.main_agent_chat_bot_v2 import main_chatbot_route_v2
+from vector_db import qdrant_crud
+
+from vector_db.qdrant_crud import Qdrant_router
+from agent_tools.autovis_tool import Autoviz_router
+from main_chat.main_chat import main_chat_router_v3
+from DB_store_backup.agentic_context_convoid_management import Db_store_router
+# Suppress warnings
+warnings.filterwarnings("ignore", message="Qdrant client version.*is incompatible.*")
+
+app = FastAPI(title="Combined AI Agent with Qdrant Collections and Redis Session Management")
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+print([route.path for route in app.routes])
+# Include routers
+
+app.include_router(main_chatbot_route)
+app.include_router(Report_Generation_Router)
+app.include_router(Redis_session_router)
+app.include_router(redis_session_route_new)
+# app.include_router(Redis_session_router_old)
+app.include_router(Db_store_router)
+app.include_router(s3_bucket_router)
+app.include_router(Autoviz_router)
+app.include_router(main_chat_router_v3)
+
+
+
+
+
+app.include_router(Qdrant_router)
+app.include_router(RetrievalQA_router)
+app.include_router(main_chatbot_route_v2)
+#==================================login and user management routes==================================
+app.include_router(login_apis_router)
+
+
+# ------------------- MIDDLEWARE -------------------
+
+@app.middleware("http")
+async def add_success_flag(request: Request, call_next):
+    response = await call_next(request)
+
+    # Only modify JSON responses
+    if "application/json" in response.headers.get("content-type", ""):
+        try:
+            body = b"".join([chunk async for chunk in response.body_iterator])
+            data = json.loads(body.decode())
+
+            # Add success flag
+            data["success"] = 200 <= response.status_code < 300
+
+            # Build new JSONResponse (auto handles Content-Length)
+            response = JSONResponse(
+                content=data,
+                status_code=response.status_code,
+                headers={k: v for k, v in response.headers.items() if k.lower() != "content-length"},
+            )
+        except Exception:
+            # fallback if response is not JSON parseable
+            pass
+    return response
+
+