fix embedding data

app.py

@@ -1,4 +1,4 @@
-from flask import Flask, jsonify
+from flask import Flask, jsonify, request
 from flask_cors import CORS
 import logging
 import os
@@ -6,13 +6,20 @@ import threading
 from dotenv import load_dotenv
 from flask_jwt_extended import JWTManager
 import datetime
-from api.admin import admin_routes
 
-# Cấu hình logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
+# Cấu hình logging cho HuggingFace
+if os.getenv("SPACE_ID"):
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        handlers=[logging.StreamHandler()]
+    )
+else:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
 logger = logging.getLogger(__name__)
 
 # Tải biến môi trường
@@ -33,31 +40,14 @@ app.config['MAX_CONTENT_LENGTH'] = 50 * 1024 * 1024  # 50MB max file size
 
 jwt = JWTManager(app)
 
-# Cho phép CORS từ frontend React với credentials
+# Cho phép CORS
 CORS(app, resources={
     r"/api/*": {
-        "origins": ["http://localhost:5173", "http://localhost:3000"], 
+        "origins": "*",  # Cho phép tất cả origins trên HF
         "supports_credentials": True
     }
 })
 
-# Đăng ký các API endpoints cho user
-from api.auth import auth_routes
-from api.chat import chat_routes
-from api.data import data_routes
-from api.history import history_routes
-from api.feedback import feedback_routes
-
-# Đăng ký các blueprint cho user
-app.register_blueprint(auth_routes, url_prefix='/api/auth')
-app.register_blueprint(chat_routes, url_prefix='/api')
-app.register_blueprint(data_routes, url_prefix='/api')
-app.register_blueprint(history_routes, url_prefix='/api')
-app.register_blueprint(feedback_routes, url_prefix='/api')
-
-# Đăng ký các blueprint cho admin
-app.register_blueprint(admin_routes, url_prefix='/api/admin')
-
 def setup_data_background():
     """Setup data in background thread"""
     try:
@@ -67,6 +57,22 @@ def setup_data_background():
         logger.info("Background data setup completed")
     except Exception as e:
         logger.error(f"Background setup failed: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+
+# Basic routes first
+@app.route('/', methods=['GET'])
+def root():
+    """Root endpoint"""
+    return jsonify({
+        "message": "Nutribot API is running",
+        "status": "healthy",
+        "endpoints": {
+            "health": "/api/health",
+            "data_status": "/api/data-status",
+            "embed_data": "/api/embed-data"
+        }
+    })
 
 @app.route('/api/health', methods=['GET'])
 def health_check():
@@ -84,7 +90,8 @@ def health_check():
         "status": "healthy",
         "message": "Server đang hoạt động",
         "time": time.strftime('%Y-%m-%d %H:%M:%S'),
-        "data_items": collection_count
+        "data_items": collection_count,
+        "environment": "huggingface" if os.getenv("SPACE_ID") else "local"
     })
 
 @app.route('/api/data-status', methods=['GET'])
@@ -120,40 +127,14 @@ def data_status():
             "error": str(e)
         }), 500
 
-@app.route('/api/admin/init', methods=['POST'])
-def init_admin():
-    """API endpoint để khởi tạo admin đầu tiên"""
-    try:
-        from models.admin_model import AdminUser
-        
-        success, result = AdminUser.create_default_super_admin()
-        
-        if success:
-            return jsonify({
-                "success": True,
-                "message": "Khởi tạo admin thành công",
-                "admin_info": result
-            })
-        else:
-            return jsonify({
-                "success": False,
-                "error": result
-            }), 400
-            
-    except Exception as e:
-        logger.error(f"Lỗi khởi tạo admin: {str(e)}")
-        return jsonify({
-            "success": False,
-            "error": str(e)
-        }), 500
-
 @app.route('/api/embed-data', methods=['POST'])
 def manual_embed_data():
     """API endpoint để chạy embedding data thủ công"""
     try:
-        from scripts.embed_data import embed_all_data
+        force = False
+        if request.is_json and request.json:
+            force = request.json.get('force', False)
         
-        force = request.json.get('force', False) if request.is_json else False
         data_dir = "data"
         
         if not os.path.exists(data_dir):
@@ -165,8 +146,12 @@ def manual_embed_data():
         # Chạy embedding trong thread riêng để không block request
         def run_embedding():
             try:
-                embed_all_data(data_dir, force=force)
-                logger.info("Manual embedding completed successfully")
+                from scripts.embed_data import embed_all_data
+                success = embed_all_data(data_dir, force=force)
+                if success:
+                    logger.info("Manual embedding completed successfully")
+                else:
+                    logger.error("Manual embedding failed")
             except Exception as e:
                 logger.error(f"Manual embedding failed: {e}")
         
@@ -184,15 +169,37 @@ def manual_embed_data():
             "error": str(e)
         }), 500
 
+# Import và đăng ký các blueprint sau khi định nghĩa basic routes
+try:
+    from api.auth import auth_routes
+    from api.chat import chat_routes
+    from api.data import data_routes
+    from api.history import history_routes
+    from api.feedback import feedback_routes
+    from api.admin import admin_routes
+
+    # Đăng ký các blueprint
+    app.register_blueprint(auth_routes, url_prefix='/api/auth')
+    app.register_blueprint(chat_routes, url_prefix='/api')
+    app.register_blueprint(data_routes, url_prefix='/api')
+    app.register_blueprint(history_routes, url_prefix='/api')
+    app.register_blueprint(feedback_routes, url_prefix='/api')
+    app.register_blueprint(admin_routes, url_prefix='/api/admin')
+    
+    logger.info("All blueprints registered successfully")
+    
+except Exception as e:
+    logger.error(f"Error importing/registering blueprints: {e}")
+
 # Auto setup data khi chạy trên HuggingFace
-if os.getenv("SPACE_ID"):  # Chỉ chạy trên HuggingFace
+if os.getenv("SPACE_ID"):
     logger.info("Detected HuggingFace environment, starting background data setup...")
     threading.Thread(target=setup_data_background, daemon=True).start()
 else:
     logger.info("Running in local environment")
 
 if __name__ == '__main__':
-    # Tạo super admin mặc định nếu chưa có
+    # Tạo admin và feedback indexes
     try:
         from models.admin_model import AdminUser
         success, result = AdminUser.create_default_super_admin()
@@ -204,7 +211,6 @@ if __name__ == '__main__':
     except Exception as e:
         logger.error(f"Lỗi tạo super admin: {e}")
 
-    # Tạo indexes cho feedback
     try:
         from models.feedback_model import ensure_indexes
         ensure_indexes()
@@ -212,6 +218,5 @@ if __name__ == '__main__':
         logger.error(f"Lỗi tạo feedback indexes: {e}")
     
     # Chạy Flask app
-    port = int(os.getenv("PORT", 7860))  # HuggingFace sử dụng port 7860
-    app.run(host='0.0.0.0', port=port, debug=False)
-    
+    port = int(os.getenv("PORT", 7860))
+    app.run(host='0.0.0.0', port=port, debug=False)

scripts/embed_data.py

@@ -2,7 +2,6 @@ import os
 import sys
 import time
 import argparse
-import logging
 
 # Set UTF-8 encoding cho console
 os.environ['PYTHONIOENCODING'] = 'utf-8'
@@ -13,83 +12,104 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from core.data_processor import DataProcessor
 from core.embedding_model import get_embedding_model
 
-# Cấu hình logging với UTF-8
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(),
-        logging.FileHandler("embed_data.log", encoding='utf-8')
-    ]
-)
-logger = logging.getLogger("embed_data")
+# Cấu hình logging phù hợp với environment
+def setup_logging():
+    import logging
+    
+    if os.getenv("SPACE_ID"):
+        # Trên HuggingFace, chỉ log ra console
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[logging.StreamHandler()]
+        )
+    else:
+        # Local environment
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.StreamHandler(),
+                logging.FileHandler("embed_data.log", encoding='utf-8')
+            ]
+        )
+    
+    return logging.getLogger("embed_data")
+
+logger = setup_logging()
 
 def embed_all_data(data_dir, force=False):
     """
     Embedding tất cả dữ liệu từ thư mục data
-    
-    Args:
-        data_dir: Đường dẫn đến thư mục chứa dữ liệu
-        force: Nếu True, sẽ xóa và tạo lại chỉ mục hiện có
     """
     logger.info(f"Bat dau qua trinh embedding du lieu tu {data_dir}")
     start_time = time.time()
     
-    # Khởi tạo các components
-    data_processor = DataProcessor(data_dir=data_dir)
-    embedding_model = get_embedding_model()
-    
-    # Kiểm tra xem có chỉ mục hiện có không
-    collection_size = embedding_model.count()
-    if collection_size > 0 and not force:
-        logger.info(f"Da ton tai chi muc voi {collection_size} items")
-        end_time = time.time()
-        logger.info(f"Hoan thanh kiem tra chi muc trong {end_time - start_time:.2f} giay")
-        logger.info("Su dung --force de tao lai chi muc")
-        return
-    
-    # Nếu buộc tạo lại hoặc chưa có chỉ mục, tạo mới
-    if force:
-        logger.info("Xoa chi muc cu va tao lai...")
-        try:
-            embedding_model.chroma_client.delete_collection(name=embedding_model.collection.name)
-            # Tạo lại collection mới
-            embedding_model.collection = embedding_model.chroma_client.create_collection(name=embedding_model.collection.name)
-            logger.info("Da xoa va tao lai collection")
-        except Exception as e:
-            logger.error(f"Loi khi xoa collection: {e}")
-    
-    # Chuẩn bị dữ liệu cho embedding
-    logger.info("Dang chuan bi du lieu cho qua trinh embedding...")
-    all_items = data_processor.prepare_for_embedding()
-    logger.info(f"Da chuan bi {len(all_items)} items de embedding")
-    
-    # Thống kê các loại dữ liệu
-    text_chunks = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "text"])
-    tables = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "table"])
-    figures = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "figure"])
-    
-    logger.info(f"Bao gom: {text_chunks} van ban, {tables} bang bieu, {figures} hinh anh")
-    
-    # Thực hiện embedding
-    logger.info("Bat dau qua trinh embedding...")
-    batch_size = 50  # Batch size phù hợp, tùy chỉnh nếu cần
-    
-    # Chia thành các batch nhỏ để xử lý
-    for i in range(0, len(all_items), batch_size):
-        batch = all_items[i:i + batch_size]
-        logger.info(f"Dang xu ly batch {i//batch_size + 1}/{(len(all_items) + batch_size - 1)//batch_size}, kich thuoc: {len(batch)}")
-        success = embedding_model.index_chunks(batch)
+    try:
+        # Khởi tạo các components
+        logger.info("Khoi tao data processor...")
+        data_processor = DataProcessor(data_dir=data_dir)
+        
+        logger.info("Khoi tao embedding model...")
+        embedding_model = get_embedding_model()
+        
+        # Kiểm tra xem có chỉ mục hiện có không
+        collection_size = embedding_model.count()
+        logger.info(f"Kich thuoc collection hien tai: {collection_size}")
+        
+        if collection_size > 0 and not force:
+            logger.info(f"Da ton tai chi muc voi {collection_size} items")
+            end_time = time.time()
+            logger.info(f"Hoan thanh kiem tra chi muc trong {end_time - start_time:.2f} giay")
+            return True
+        
+        # Nếu buộc tạo lại hoặc chưa có chỉ mục, tạo mới
+        if force and collection_size > 0:
+            logger.info("Xoa chi muc cu va tao lai...")
+            try:
+                embedding_model.delete_collection()
+                logger.info("Da xoa va tao lai collection")
+            except Exception as e:
+                logger.error(f"Loi khi xoa collection: {e}")
+        
+        # Chuẩn bị dữ liệu cho embedding
+        logger.info("Dang chuan bi du lieu cho qua trinh embedding...")
+        all_items = data_processor.prepare_for_embedding()
+        logger.info(f"Da chuan bi {len(all_items)} items de embedding")
+        
+        if not all_items:
+            logger.warning("Khong co items nao de embedding")
+            return False
+        
+        # Thống kê các loại dữ liệu
+        text_chunks = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "text"])
+        tables = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "table"])
+        figures = len([item for item in all_items if item.get("metadata", {}).get("content_type") == "figure"])
+        
+        logger.info(f"Bao gom: {text_chunks} van ban, {tables} bang bieu, {figures} hinh anh")
+        
+        # Thực hiện embedding
+        logger.info("Bat dau qua trinh embedding...")
+        success = embedding_model.index_chunks(all_items)
+        
         if not success:
-            logger.error(f"Loi xu ly batch {i//batch_size + 1}")
-            return
-    
-    end_time = time.time()
-    elapsed_time = end_time - start_time
-    
-    # Kiểm tra kết quả cuối cùng
-    final_count = embedding_model.count()
-    logger.info(f"Hoan thanh qua trinh embedding {final_count} items trong {elapsed_time:.2f} giay")
+            logger.error("Loi xu ly embedding")
+            return False
+        
+        end_time = time.time()
+        elapsed_time = end_time - start_time
+        
+        # Kiểm tra kết quả cuối cùng
+        final_count = embedding_model.count()
+        logger.info(f"Hoan thanh qua trinh embedding {final_count} items trong {elapsed_time:.2f} giay")
+        
+        return True
+        
+    except Exception as e:
+        logger.error(f"Loi embedding: {str(e)}")
+        import traceback
+        logger.error(traceback.format_exc())
+        return False
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Embedding du lieu cho he thong Nutribot")
@@ -109,4 +129,10 @@ if __name__ == "__main__":
         sys.exit(1)
     
     # Thực hiện embedding
-    embed_all_data(data_dir, args.force)
+    success = embed_all_data(data_dir, args.force)
+    if success:
+        logger.info("Embedding hoan thanh thanh cong!")
+        sys.exit(0)
+    else:
+        logger.error("Embedding that bai!")
+        sys.exit(1)

startup.py

@@ -3,39 +3,72 @@ import sys
 import logging
 from pathlib import Path
 
-# Setup logging
-logging.basicConfig(level=logging.INFO)
+# Setup logging cho HuggingFace environment
+def setup_logging():
+    """Setup logging phù hợp với HF environment"""
+    if os.getenv("SPACE_ID"):
+        # Trên HF, chỉ log ra console
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[logging.StreamHandler()]
+        )
+    else:
+        # Local, có thể ghi file
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.StreamHandler(),
+                logging.FileHandler("embed_data.log", encoding='utf-8')
+            ]
+        )
+
+setup_logging()
 logger = logging.getLogger(__name__)
 
 def setup_data():
     """Setup and embed data on startup"""
     try:
-        # Import các module cần thiết
-        from scripts.embed_data import embed_all_data
+        logger.info("Starting data setup process...")
+        
+        # Kiểm tra data directory
+        data_dir = "data"
+        if not os.path.exists(data_dir):
+            logger.error(f"Data directory {data_dir} not found!")
+            return
+        
+        # Import sau khi đã setup logging
+        logger.info("Importing embedding modules...")
         from core.embedding_model import get_embedding_model
         
         # Kiểm tra xem đã có data chưa
+        logger.info("Checking existing embeddings...")
         embedding_model = get_embedding_model()
         current_count = embedding_model.count()
         
         logger.info(f"Current embeddings count: {current_count}")
         
         # Nếu chưa có data hoặc ít hơn expected, thì embed
-        if current_count < 100:  # Adjust threshold as needed
+        if current_count < 50:  # Threshold thấp hơn để test
             logger.info("Starting data embedding process...")
-            data_dir = "data"
             
-            if os.path.exists(data_dir):
-                embed_all_data(data_dir, force=False)
-                logger.info("Data embedding completed!")
-            else:
-                logger.error(f"Data directory {data_dir} not found!")
+            # Import embed function
+            from scripts.embed_data import embed_all_data
+            
+            # Chạy embedding
+            embed_all_data(data_dir, force=False)
+            
+            # Kiểm tra lại
+            final_count = embedding_model.count()
+            logger.info(f"Embedding completed! Final count: {final_count}")
         else:
             logger.info("Data already embedded, skipping...")
             
     except Exception as e:
         logger.error(f"Error in setup_data: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
 
 if __name__ == "__main__":
-    setup_data()
-    
+    setup_data()