import argparse import chromadb from tqdm import tqdm # Optional: For progress bar db_config = { "youtube_db": { "source_db_path": "../youtube_surfer_ai_agent/youtube_db", "source_collection_name": "yt_metadata", "destination_collection_name": "yt_metadata", }, "divya_prabandham": { "source_db_path": "../uveda_analyzer/chromadb_store", "source_collection_name": "divya_prabandham", "destination_collection_name": "divya_prabandham", }, "divya_prabandham_taniyans": { "source_db_path": "../uveda_analyzer/chromadb_store", "source_collection_name": "divya_prabandham_taniyans", "destination_collection_name": "divya_prabandham_taniyans", }, "vishnu_sahasranamam": { "source_db_path": "../vishnu_sahasranamam_ai/output/chroma_store", "source_collection_name": "vishnu_sahasranamam", "destination_collection_name": "vishnu_sahasranamam_openai", }, "bhagavat_gita": { "source_db_path": "../bhagavat_gita_chat/chromadb_store", "source_collection_name": "bhagavat_gita", "destination_collection_name": "bhagavat_gita_openai", }, "pancha_sooktham": { "source_db_path": "../sooktham_ai/chromadb_store", "source_collection_name": "pancha_sooktham", "destination_collection_name": "pancha_sooktham", }, "taitriya_upanishad": { "source_db_path": "../taitriya_upanishad_ai/chromadb_store", "source_collection_name": "taitriya_upanishad", "destination_collection_name": "taitriya_upanishad", }, "shanthi_panchakam": { "source_db_path": "../shanthi_panchakam_ai/chromadb_store", "source_collection_name": "shanthi_panchakam", "destination_collection_name": "shanthi_panchakam", }, "taitriya_samhitha": { "source_db_path": "../taitriya_samhitha_ai/chromadb_store", "source_collection_name": "taitriya_samhitha", "destination_collection_name": "taitriya_samhitha", }, "taitriya_brahmanam": { "source_db_path": "../taitriya_brahmanam_ai/chromadb_store", "source_collection_name": "taitriya_brahmanam", "destination_collection_name": "taitriya_brahmanam", }, "katakam": { "source_db_path": "../taitriya_brahmanam_ai/chromadb_store", "source_collection_name": "katakam", "destination_collection_name": "katakam", }, "sri_stavam": { "source_db_path": "../vedam_ai/chromadb-store", "source_collection_name": "sri_stavam", "destination_collection_name": "sri_stavam", }, "taitriya_aranyakam": { "source_db_path": "../taitriya_aranyakam_ai/chromadb_store", "source_collection_name": "taitriya_aranyakam", "destination_collection_name": "taitriya_aranyakam", }, "brahma_sutra": { "source_db_path": "../brahma_sutra_ai/chromadb_store", "source_collection_name": "brahma_sutra", "destination_collection_name": "brahma_sutra", }, "valmiki_ramayanam": { "source_db_path": "../valmiki_ramayanam_ai/chromadb_store", "source_collection_name": "valmiki_ramayanam", "destination_collection_name": "valmiki_ramayanam_openai", }, "sri_vachana_bhushanam": { "source_db_path": "../sri_vachana_bhushanam_ai/chromadb_store", "source_collection_name": "sri_vachana_bhushanam", "destination_collection_name": "sri_vachana_bhushanam", }, "desika_prabandham": { "source_db_path": "../desika_prabandham_ai/chromadb_store", "source_collection_name": "desika_prabandham", "destination_collection_name": "desika_prabandham", }, "raghuveera_gadhyam": { "source_db_path": "../raghuveera_gadhyam_ai/chromadb_store", "source_collection_name": "raghuveera_gadhyam", "destination_collection_name": "raghuveera_gadhyam", }, "narayaneeyam": { "source_db_path": "../narayaneeyam_ai/chromadb_store", "source_collection_name": "narayaneeyam", "destination_collection_name": "narayaneeyam", }, "bhagavata_purana": { "source_db_path": "../bhagavata_purana_ai/chromadb_store", "source_collection_name": "bhagavata_purana", "destination_collection_name": "bhagavata_purana", }, "agnipuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "agnipuranam", "destination_collection_name": "agnipuranam" }, "bhavishyapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "bhavishyapuranam", "destination_collection_name": "bhavishyapuranam" }, "brahmandpuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "brahmandpuranam", "destination_collection_name": "brahmandpuranam" }, "brahmapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "brahmapuranam", "destination_collection_name": "brahmapuranam" }, "brahmavaivarthapurana": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "brahmavaivarthapurana", "destination_collection_name": "brahmavaivarthapurana" }, "garudapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "garudapuranam", "destination_collection_name": "garudapuranam" }, "harivanshapuraanam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "harivanshapuraanam", "destination_collection_name": "harivanshapuraanam" }, "kurmapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "kurmapuranam", "destination_collection_name": "kurmapuranam" }, "lingapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "lingapuranam", "destination_collection_name": "lingapuranam" }, "markandeypuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "markandeypuranam", "destination_collection_name": "markandeypuranam" }, "matsyapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "matsyapuranam", "destination_collection_name": "matsyapuranam" }, "naradapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "naradapuranam", "destination_collection_name": "naradapuranam" }, "padmapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "padmapuranam", "destination_collection_name": "padmapuranam" }, "shivapuraanam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "shivapuraanam", "destination_collection_name": "shivapuraanam" }, "skandapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "skandapuranam", "destination_collection_name": "skandapuranam" }, "vaamanapuraanam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "vaamanapuraanam", "destination_collection_name": "vaamanapuraanam" }, "vaayupuraanam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "vaayupuraanam", "destination_collection_name": "vaayupuraanam" }, "varahapuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "varahapuranam", "destination_collection_name": "varahapuranam" }, "vishnupuranam": { "source_db_path": "../puranas_ai/chromadb_store", "source_collection_name": "vishnupuranam", "destination_collection_name": "vishnu_puranam_openai" }, "vaazhi_thirunaamams": { "source_db_path": "../vaazhi_thirunamam/chromadb_store", "source_collection_name": "vaazhi_thirunaamams", "destination_collection_name": "vaazhi_thirunaamams" }, "upadesa_rathnamalai": { "source_db_path": "../upadesa_rathnamalai/chromadb_store", "source_collection_name": "upadesa_rathnamalai", "destination_collection_name": "upadesa_rathnamalai" }, "thiruvaimozhi_nootrandhadhi": { "source_db_path": "../thiruvaimozhi_nootrandhadhi_ai/chromadb_store", "source_collection_name": "thiruvaimozhi_nootrandhadhi", "destination_collection_name": "thiruvaimozhi_nootrandhadhi" }, "devaraja_ashtakam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "devaraja_ashtakam", "destination_collection_name": "devaraja_ashtakam" }, "geethartha_sangraha": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "geethartha_sangraha", "destination_collection_name": "geethartha_sangraha" }, "mukunda_mala": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "mukunda_mala", "destination_collection_name": "mukunda_mala" }, "narasimha_ashtakam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "narasimha_ashtakam", "destination_collection_name": "narasimha_ashtakam" }, "panchayudha_stotram": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "panchayudha_stotram", "destination_collection_name": "panchayudha_stotram" }, "ranganatha_stotram": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "ranganatha_stotram", "destination_collection_name": "ranganatha_stotram" }, "devaraja_mangalam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "devaraja_mangalam", "destination_collection_name": "devaraja_mangalam" }, "dhati_panchakam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "dhati_panchakam", "destination_collection_name": "dhati_panchakam" }, "mukthaka_mangalam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "mukthaka_mangalam", "destination_collection_name": "mukthaka_mangalam" }, "venkateswara_stotram": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "venkateswara_stotram", "destination_collection_name": "venkateswara_stotram" }, "yathiraja_vimsathi": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "yathiraja_vimsathi", "destination_collection_name": "yathiraja_vimsathi" }, "gadhyathrayam": { "source_db_path": "../stotra_patam_ai/chromadb_store", "source_collection_name": "gadhyathrayam", "destination_collection_name": "gadhyathrayam" }, "aarthi_prabandham": { "source_db_path": "../aarthi_prabandham_ai/chromadb_store", "source_collection_name": "aarthi_prabandham", "destination_collection_name": "aarthi_prabandham" }, "padhuka_sahasram": { "source_db_path": "../padhuka_sahasram/chromadb_store", "source_collection_name": "padhuka_sahasram", "destination_collection_name": "padhuka_sahasram" }, } parser = argparse.ArgumentParser(description="My app with database parameter") parser.add_argument( "--db", type=str, required=True, choices=list(db_config.keys()), help=f"Id of the database to use. allowed_values : {', '.join(db_config.keys())}", ) args = parser.parse_args() db_id = args.db if db_id is None: raise Exception(f"No db provided!") if db_id not in db_config: raise Exception(f"db with id {db_id} not found!") # Connect to source and destination local persistent clients source_client = chromadb.PersistentClient(path=db_config[db_id]["source_db_path"]) destination_client = chromadb.PersistentClient(path="./chromadb-store") source_collection_name = db_config[db_id]["source_collection_name"] destination_collection_name = db_config[db_id]["destination_collection_name"] # Get the source collection source_collection = source_client.get_collection(source_collection_name) # Retrieve all data from the source collection source_data = source_collection.get(include=["documents", "metadatas", "embeddings"]) # Create or get the destination collection if destination_client.get_or_create_collection(destination_collection_name): print("Deleting existing collection", destination_collection_name) destination_client.delete_collection(destination_collection_name) destination_collection = destination_client.get_or_create_collection( destination_collection_name, metadata=source_collection.metadata, # Copy metadata if needed ) # Add data to the destination collection in batches BATCH_SIZE = 500 total_records = len(source_data["ids"]) print(f"Copying {total_records} records in batches of {BATCH_SIZE}...") for i in tqdm(range(0, total_records, BATCH_SIZE)): batch_ids = source_data["ids"][i : i + BATCH_SIZE] batch_docs = source_data["documents"][i : i + BATCH_SIZE] batch_metas = source_data["metadatas"][i : i + BATCH_SIZE] batch_embeds = ( source_data["embeddings"][i : i + BATCH_SIZE] if "embeddings" in source_data and source_data["embeddings"] is not None else None ) destination_collection.add( ids=batch_ids, documents=batch_docs, metadatas=batch_metas, embeddings=batch_embeds, ) print("✅ Collection copied successfully!") print("Total records in source collection = ", source_collection.count()) print("Total records in destination collection = ", destination_collection.count())