update app

.gitignore

@@ -1,3 +1,4 @@
 \demovv
 BoPhapDienDienTu
-vbpl_links.txt
+vbpl_links.txt
+.env

app.py

@@ -1,4 +1,5 @@
 import os
+from dotenv import load_dotenv
 import shutil
 import numpy as np
 from pathlib import Path
@@ -12,6 +13,11 @@ from torch.utils.data import DataLoader
 from accelerate import Accelerator
 from datasets import Dataset
 
+load_dotenv()
+
+hf_token = os.getenv("HF_TOKEN")
+if hf_token is None:
+    raise ValueError("HF_TOKEN not in the .env file")
 
 # Wrapper for embedding
 class SentenceTransformerWrapper:
@@ -156,9 +162,6 @@ def save_to_chromadb(
     vector_db.persist()
     print(f"Database saved successfully to {persist_directory}")
 
-    shutil.make_archive("chroma_db", "zip", "./chroma_db")
-    print("Vector database archived as chroma_db.zip")
-
     return vector_db
 
 
@@ -187,9 +190,15 @@ if __name__ == "__main__":
     # Step 6: Generate embeddings and save to ChromaDB
     save_to_chromadb(processed_docs, processed_metadata, embedding_model)
 
-    # os.system("git lfs install")
-    os.system("git add chroma_db/")
-    os.system("git commit -m 'Persist vector database after processing'")
-    os.system("git push")
-    os.system("git config --global user.email '[email protected]'")
-    os.system("git config --global user.name 'My Duong'")
+    shutil.make_archive("chroma_db", "zip", "./chroma_db")
+    print("Vector database archived as chroma_db.zip")
+
+    with open("chroma_db.zip", "rb") as f:
+        zip_bytes = f.read()
+
+    # Create a dataset from the zip file bytes
+    dataset = Dataset.from_dict({"vector_db_zip": [zip_bytes]})
+
+    # Push to the HF Datasets Hub (replace with your username, repo name, and use a valid token)
+    dataset.push_to_hub("camiellia/phapdien_demo", token=hf_token)
+    print("Vector database uploaded to the HF Datasets Hub.")