change logic from sentence name to representative name

auth.py

@@ -5,7 +5,7 @@ from fastapi.security import OAuth2PasswordBearer
 from passlib.context import CryptContext
 from config import SECRET_KEY, ALGORITHM, ACCESS_TOKEN_EXPIRE_HOURS
 from models import TokenData, UserInDB, User
-from database import users_db
+from database import get_users
 from typing import Annotated, Optional
 from jwt.exceptions import InvalidTokenError
 
@@ -54,7 +54,7 @@ async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]):
         token_data = TokenData(username=username)
     except InvalidTokenError:
         raise credentials_exception
-    user = get_user(users_db, username=token_data.username)
+    user = get_user(get_users(), username=token_data.username)
     if user is None:
         raise credentials_exception
     return user

config.py

@@ -14,6 +14,7 @@ SUBJECT_DATA_FILE = os.path.join(DATA_DIR, "subjectData.csv")
 SAMPLE_DATA_FILE = os.path.join(DATA_DIR, "sampleData.csv")
 # Model Names
 MODEL_NAME = "Detomo/cl-nagoya-sup-simcse-ja-for-standard-name-v1_0"
-SETENCE_EMBEDDING_FILE = os.path.join(DATA_DIR, "sample_name_sentence_embeddings(cl-nagoya-sup-simcse-ja-for-standard-name-v1_1).pkl")
-SETENCE_SIMILARITY_FILE = os.path.join(DATA_DIR, "sample_name_sentence_similarities(cl-nagoya-sup-simcse-ja-for-standard-name-v1_1).pkl")
-
+SENTENCE_EMBEDDING_FILE = os.path.join(
+    DATA_DIR,
+    "sample_representative_name_sentence_embeddings(cl-nagoya-sup-simcse-ja-for-standard-name-v1_0).pkl",
+)

data/sample_name_sentence_similarities(cl-nagoya-sup-simcse-ja-for-standard-name-v1_1).pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d942620d2940849fdee0f6cec443a5dd1f7f608144d4f1cee5ff66dd39797035
-size 137593306

data/{sample_name_sentence_similarities(cl-nagoya-sup-simcse-ja-base).pkl → sample_representative_name_sentence_embeddings(cl-nagoya-sup-simcse-ja-for-standard-name-v1_0).pkl}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a50b8dd5d6625d23fdf960eaaecf550fd3119c9267ef5e7408dae99c1a16577f
-size 133819030
+oid sha256:aa42e9df217a42147a4a1e2a584a35462756f9f34646a6db981941cf89dc2095
+size 18217123

data/users.json

@@ -0,0 +1,16 @@
+{
+    "chien_vm": {
+        "username": "chien_vm",
+        "full_name": "Chien VM",
+        "email": "[email protected]",
+        "hashed_password": "$2b$12$RtcKFk7B3hKd7vYkwxdFN.eBXSiryQIRUG.OoJ07Pl9lzHNUkugMi",
+        "disabled": false
+    },
+    "meiseidev": {
+        "username": "meiseidev",
+        "full_name": "meiseidev",
+        "email": "",
+        "hashed_password": "$2b$12$LXIbdQ388dMiN/ej76zuHeHuuz.VHz9rJfH4FpwdVbqfwCbSI55Va",
+        "disabled": false
+    }
+}

database.py

@@ -1,16 +1,73 @@
-users_db = {
-    "chien_vm": {
-        "username": "chien_vm",
-        "full_name": "Chien VM",
-        "email": "[email protected]",
-        "hashed_password": "$2b$12$RtcKFk7B3hKd7vYkwxdFN.eBXSiryQIRUG.OoJ07Pl9lzHNUkugMi",
-        "disabled": False,
-    },
-    "hoi_nv": {
-        "username": "hoi_nv",
-        "full_name": "Hoi NV",
-        "email": "[email protected]",
-        "hashed_password": "$2b$12$RtcKFk7B3hKd7vYkwxdFN.eBXSiryQIRUG.OoJ07Pl9lzHNUkugMi",
+import json
+import os
+from passlib.context import CryptContext
+
+# Tạo thư mục data nếu chưa tồn tại
+os.makedirs("data", exist_ok=True)
+USERS_FILE = "data/users.json"
+
+# Context để hash password
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+# Hàm để đọc users từ file JSON
+def get_users():
+    if not os.path.exists(USERS_FILE):
+        # Tạo file với dữ liệu mặc định nếu chưa tồn tại
+        default_users = {
+            "chien_vm": {
+                "username": "chien_vm",
+                "full_name": "Chien VM",
+                "email": "[email protected]",
+                "hashed_password": "$2b$12$RtcKFk7B3hKd7vYkwxdFN.eBXSiryQIRUG.OoJ07Pl9lzHNUkugMi",
+                "disabled": False,
+            },
+            "hoi_nv": {
+                "username": "hoi_nv",
+                "full_name": "Hoi NV",
+                "email": "[email protected]",
+                "hashed_password": "$2b$12$RtcKFk7B3hKd7vYkwxdFN.eBXSiryQIRUG.OoJ07Pl9lzHNUkugMi",
+                "disabled": False,
+            },
+        }
+        save_users(default_users)
+        return default_users
+
+    with open(USERS_FILE, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+# Hàm để lưu users vào file JSON
+def save_users(users_data):
+    with open(USERS_FILE, "w", encoding="utf-8") as f:
+        json.dump(users_data, f, indent=4, ensure_ascii=False)
+
+
+# Hàm để tạo tài khoản mới
+def create_account(username, password):
+    # Kiểm tra xem username đã tồn tại chưa
+    users = get_users()
+    if username in users:
+        return False, "Username already exists"
+
+    # Hash password
+    hashed_password = pwd_context.hash(password)
+
+    # Tạo user mới
+    new_user = {
+        "username": username,
+        "full_name": username,  # Mặc định full_name là username
+        "email": "",  # Không yêu cầu email
+        "hashed_password": hashed_password,
         "disabled": False,
     }
-}
+
+    # Thêm user mới vào database
+    users[username] = new_user
+    save_users(users)
+
+    return True, "Account created successfully"
+
+
+# Để tương thích với code cũ
+users_db = get_users()

main.py

@@ -36,7 +36,7 @@ app = FastAPI(
     openapi_tags=[
         {"name": "Health", "description": "Health check endpoints"},
         {"name": "Authentication", "description": "User authentication and token management"},
-        {"name": "Prediction", "description": " Predict and process CSV files"},
+        {"name": "Prediction", "description": "Predict and process CSV files"},
     ]
 )
 

models.py

@@ -16,3 +16,8 @@ class User(BaseModel):
 
 class UserInDB(User):
     hashed_password: str
+
+
+class UserCreate(BaseModel):
+    username: str
+    password: str

routes/auth.py

@@ -2,9 +2,9 @@ from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordRequestForm
 from datetime import timedelta
 from auth import authenticate_user, create_access_token
-from models import Token
+from models import Token, UserCreate
 from config import ACCESS_TOKEN_EXPIRE_HOURS
-from database import users_db
+from database import get_users, create_account
 
 router = APIRouter()
 
@@ -13,7 +13,7 @@ async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(
     """
     Endpoint để lấy access token bằng username và password
     """
-    user = authenticate_user(users_db, form_data.username, form_data.password)
+    user = authenticate_user(get_users(), form_data.username, form_data.password)
     if not user:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
@@ -26,3 +26,14 @@ async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends(
         data={"sub": user.username}, expires_delta=access_token_expires
     )
     return Token(access_token=access_token, token_type="bearer")
+
+
+@router.post("/register")
+async def register_user(user_data: UserCreate):
+    """
+    Endpoint để đăng ký tài khoản mới
+    """
+    success, message = create_account(user_data.username, user_data.password)
+    if not success:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message)
+    return {"message": message}

routes/predict.py

@@ -7,6 +7,7 @@ from fastapi.responses import FileResponse
 from auth import get_current_user
 from services.sentence_transformer_service import SentenceTransformerService, sentence_transformer_service
 from data_lib.input_name_data import InputNameData
+from data_lib.base_data import COL_NAME_SENTENCE
 from mapping_lib.name_mapping_helper import NameMappingHelper
 from config import UPLOAD_DIR, OUTPUT_DIR
 
@@ -40,14 +41,23 @@ async def predict(
         inputData = InputNameData(sentence_service.dic_standard_subject)
         inputData.load_data_from_csv(input_file_path)
         inputData.process_data()
+        input_name_sentences = inputData.dataframe[COL_NAME_SENTENCE]
+        input_name_sentence_embeddings = sentence_service.sentenceTransformerHelper.create_embeddings(input_name_sentences)
+        
+        # Create similarity matrix
+        similarity_matrix = sentence_service.sentenceTransformerHelper.create_similarity_matrix_from_embeddings(
+            sentence_service.sample_name_sentence_embeddings,
+            input_name_sentence_embeddings
+        )
 
         # Map standard names
         nameMappingHelper = NameMappingHelper(
             sentence_service.sentenceTransformerHelper,
             inputData,
             sentence_service.sampleData,
+            input_name_sentence_embeddings,
             sentence_service.sample_name_sentence_embeddings,
-            sentence_service.sample_name_sentence_similarities,
+            similarity_matrix,
         )
         df_predicted = nameMappingHelper.map_standard_names()
 

services/sentence_transformer_service.py

@@ -1,25 +1,20 @@
 import pickle
 from config import (
     MODEL_NAME, 
-    SETENCE_EMBEDDING_FILE, 
-    SETENCE_SIMILARITY_FILE, 
+    SENTENCE_EMBEDDING_FILE,  
     SAMPLE_DATA_FILE, SUBJECT_DATA_FILE
 )
 from sentence_transformer_lib.sentence_transformer_helper import SentenceTransformerHelper
 from data_lib.subject_data import SubjectData
 from data_lib.sample_name_data import SampleNameData
-from clustering_lib.sentence_clustering_lib import SentenceClusteringLib
-from data_lib.base_data import COL_STANDARD_NAME_KEY, COL_STANDARD_SUBJECT, COL_STANDARD_NAME
+
 
 class SentenceTransformerService:
     def __init__(self):
         self.sentenceTransformerHelper = None
         self.dic_standard_subject = None
         self.sample_name_sentence_embeddings = None
-        self.sample_name_sentence_similarities = None
         self.sampleData = None
-        self.sentence_clustering_lib = None
-        self.name_groups = None
 
     def load_model_data(self):
         """Load model and data only once at startup"""
@@ -38,26 +33,13 @@ class SentenceTransformerService:
         self.dic_standard_subject = SubjectData.create_standard_subject_dic_from_file(SUBJECT_DATA_FILE)
 
         # Load pre-computed embeddings and similarities
-        with open(SETENCE_EMBEDDING_FILE, "rb") as f:
+        with open(SENTENCE_EMBEDDING_FILE, "rb") as f:
             self.sample_name_sentence_embeddings = pickle.load(f)
 
-        with open(SETENCE_SIMILARITY_FILE, "rb") as f:
-            self.sample_name_sentence_similarities = pickle.load(f)
-
         # Load and process sample data
         self.sampleData = SampleNameData()
         self.sampleData.load_data_from_csv(SAMPLE_DATA_FILE)
         self.sampleData.process_data()
-
-        # Create sentence clusters
-        self.sentence_clustering_lib = SentenceClusteringLib(self.sample_name_sentence_embeddings)
-        best_name_eps = 0.07
-        self.name_groups, _ = self.sentence_clustering_lib.create_sentence_cluster(best_name_eps)
-
-        self.sampleData._create_key_column(
-            COL_STANDARD_NAME_KEY, COL_STANDARD_SUBJECT, COL_STANDARD_NAME
-        )
-        self.sampleData.set_name_sentence_labels(self.name_groups)
         self.sampleData.build_search_tree()
 
         print("Models and data loaded successfully")