change predict rule

Dockerfile

@@ -28,8 +28,8 @@ COPY requirements.txt .
 RUN --mount=type=secret,id=BITBUCKET_APP_PW,mode=0444,required=true \
     git clone https://vumichien:$(cat /run/secrets/BITBUCKET_APP_PW)@bitbucket.org/dtm-partners/meisai-check-ai.git && \
     cd meisai-check-ai && \
-    git checkout develop && \
-    git pull origin develop && \
+    git checkout staging && \
+    git pull origin staging && \
     cd ..
 
 # Cài đặt dependencies

routes/predict.py

@@ -21,8 +21,14 @@ from mapping_lib.sub_subject_and_name_data_mapper import SubSubjectAndNameDataMa
 from mapping_lib.sub_subject_location_data_mapper import SubSubjectLocationDataMapper
 from mapping_lib.abstract_similarity_mapper import AbstractSimilarityMapper
 from mapping_lib.name_and_abstract_mapper import NameAndAbstractDataMapper
-from mapping_lib.unit_similarity_mapper import UnitSimilarityMapper
-from mapping_lib.standard_name_mapper import StandardNameMapper
+from mapping_lib.unit_mapper import UnitMapper
+from mapping_lib.base_dictionary_mapper import BaseDictionaryMapper
+from common_lib.data_utilities import fillna_with_space
+from common_lib.string_utilities import (
+    preprocess_text,
+    ConversionType,
+    ConversionSettings,
+)
 
 from config import UPLOAD_DIR, OUTPUT_DIR
 from models import (
@@ -65,6 +71,21 @@ async def predict(
         # Load input data
         start_time = time.time()
         df_input_data = pd.read_csv(input_file_path)
+        
+        # Preprocess data like in meisai-check-ai/predict.py
+        df_input_data["元名称"] = df_input_data["名称"]
+        df_input_data["名称"] = df_input_data["名称"].apply(
+            lambda x: (
+                preprocess_text(
+                    x,
+                    convert_kana=ConversionType.Z2H,
+                    convert_alphabet=ConversionType.Z2H,
+                    convert_digit=ConversionType.Z2H,
+                )
+                if pd.notna(x)
+                else ""
+            )
+        )
 
         # Ensure basic columns exist with default values
         basic_columns = {
@@ -83,9 +104,8 @@ async def predict(
             if col not in df_input_data.columns:
                 df_input_data[col] = default_value
 
-        # Process data using the new mapping system similar to predict.py
+        # SubjectSimilarityMapper
         try:
-            # Subject mapping
             if sentence_service.df_subject_map_data is not None:
                 subject_similarity_mapper = SubjectSimilarityMapper(
                     cached_embedding_helper=sentence_service.subject_cached_embedding_helper,
@@ -93,35 +113,29 @@ async def predict(
                 )
 
                 list_input_subject = df_input_data["科目"].unique()
-                df_subject_data = pd.DataFrame({"科目": list_input_subject})
+                df_subject_data = pd.DataFrame(list_input_subject, columns=["科目"])
 
-                subject_similarity_mapper.predict_input_optimized(df_input_data=df_subject_data)
+                subject_similarity_mapper.predict_input(df_input_data=df_subject_data)
 
-                output_subject_map = dict(
-                    zip(df_subject_data["科目"], df_subject_data["出力_科目"])
-                )
-                df_input_data["標準科目"] = df_input_data["科目"].map(
-                    output_subject_map
-                )
-                df_input_data["出力_科目"] = df_input_data["科目"].map(
-                    output_subject_map
-                )
+                output_subject_map = dict(zip(df_subject_data["科目"], df_subject_data["出力_科目"]))
+                df_input_data["標準科目"] = df_input_data["科目"].map(output_subject_map)
+                df_input_data["出力_科目"] = df_input_data["標準科目"]
+                fillna_with_space(df_input_data)
 
         except Exception as e:
             print(f"Error processing SubjectSimilarityMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
+        # StandardSubjectDataMapper
         try:
-            # Standard subject mapping
             if sentence_service.df_standard_subject_map_data is not None:
                 standard_subject_data_mapper = StandardSubjectDataMapper(
                     df_map_data=sentence_service.df_standard_subject_map_data
                 )
                 df_output_data = standard_subject_data_mapper.map_data(
-                    df_input_data=df_input_data,
-                    input_key_columns=["出力_科目"],
-                    in_place=True,
+                    df_input_data=df_input_data, input_key_columns=["出力_科目"], in_place=True
                 )
+                fillna_with_space(df_output_data)
             else:
                 df_output_data = df_input_data.copy()
 
@@ -130,131 +144,127 @@ async def predict(
             # Continue with original data if standard subject mapping fails
             df_output_data = df_input_data.copy()
 
+        # SubSubjectSimilarityMapper
         try:
-            # Sub subject mapping
             if sentence_service.df_sub_subject_map_data is not None:
                 sub_subject_similarity_mapper = SubSubjectSimilarityMapper(
                     cached_embedding_helper=sentence_service.sub_subject_cached_embedding_helper,
                     df_map_data=sentence_service.df_sub_subject_map_data,
                 )
-                sub_subject_similarity_mapper.predict_input_optimized(
-                    df_input_data=df_output_data
+                df_input_sub_subject = df_output_data[
+                    ["科目", "標準科目", "出力_科目", "中科目", "分類"]
+                ].drop_duplicates()
+                sub_subject_similarity_mapper.predict_input(df_input_data=df_input_sub_subject)
+                
+                sub_subject_map_key_columns = ["科目", "標準科目", "出力_科目", "中科目", "分類"]
+                sub_subject_map_data_columns = [
+                    "出力_基準中科目",
+                    "出力_中科目類似度",
+                    "出力_中科目",
+                    "外部・内部区分",
+                ]
+
+                sub_subject_data_mapper = BaseDictionaryMapper(
+                    df_input_sub_subject, sub_subject_map_key_columns, sub_subject_map_data_columns
+                )
+                sub_subject_data_mapper.map_data(
+                    df_input_data=df_output_data,
+                    input_key_columns=sub_subject_map_key_columns,
+                    in_place=True,
                 )
-                df_output_data = df_output_data.fillna("")
+                fillna_with_space(df_output_data)
 
         except Exception as e:
             print(f"Error processing SubSubjectSimilarityMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
-        
+        # NameSimilarityMapper
         try:
-            # Name mapping
             if sentence_service.df_name_map_data is not None:
                 name_sentence_mapper = NameSimilarityMapper(
                     cached_embedding_helper=sentence_service.name_cached_embedding_helper,
                     df_map_data=sentence_service.df_name_map_data,
                 )
-                name_sentence_mapper.predict_input_optimized(df_input_data=df_output_data)
+                name_sentence_mapper.predict_input(df_input_data=df_output_data)
+                fillna_with_space(df_output_data)
 
         except Exception as e:
             print(f"Error processing NameSimilarityMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
-        try:
-            sub_subject_location_mapper = SubSubjectLocationDataMapper()
-            sub_subject_location_mapper.map_location(df_output_data)
-        except Exception as e:
-            print(f"Error processing SubSubjectLocationDataMapper: {e}")
-            raise HTTPException(status_code=500, detail=str(e))
 
+        # SubSubjectAndNameDataMapper
         try:
-            # Sub subject and name mapping
             if sentence_service.df_sub_subject_and_name_map_data is not None:
-                sub_subject_and_name_mapper = SubSubjectAndNameDataMapper(
+                sub_subject_and_name_data_mapper = SubSubjectAndNameDataMapper(
                     df_map_data=sentence_service.df_sub_subject_and_name_map_data
                 )
-                sub_subject_and_name_mapper.map_data(df_input_data=df_output_data)
+                sub_subject_and_name_data_mapper.map_data(df_input_data=df_output_data)
 
         except Exception as e:
             print(f"Error processing SubSubjectAndNameDataMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
+        # UnitMapper
         try:
-            # Abstract mapping
-            if sentence_service.df_abstract_map_data is not None:
-                # Ensure required columns exist before AbstractSimilarityMapper
-                required_columns_for_abstract = {
-                    "標準科目": "",
-                    "摘要グループ": "",
-                    "確定": "未確定",
-                    "摘要": "",
-                    "備考": "",
-                }
-
-                # Add missing columns with appropriate defaults
-                for col, default_val in required_columns_for_abstract.items():
-                    if col not in df_output_data.columns:
-                        df_output_data[col] = default_val
-                        print(
-                            f"DEBUG: Added missing column '{col}' with default value '{default_val}'"
-                        )
-
-                # Ensure data types are correct (convert to string to avoid type issues)
-                for col in ["標準科目", "摘要グループ", "確定", "摘要", "備考"]:
-                    if col in df_output_data.columns:
-                        df_output_data[col] = df_output_data[col].astype(str).fillna("")
+            if sentence_service.df_unit_map_data is not None:
+                unit_similarity_mapper = UnitMapper(
+                    cached_embedding_helper=sentence_service.unit_cached_embedding_helper,
+                    df_map_data=sentence_service.df_unit_map_data,
+                )
+                unit_map_key_columns = ["単位"]
+                df_input_unit = df_input_data[unit_map_key_columns].drop_duplicates()
+                unit_similarity_mapper.predict_input(df_input_data=df_input_unit)
+                
+                output_unit_data_columns = ["出力_基準単位", "出力_単位類似度", "出力_集計用単位", "出力_標準単位"]
+                unit_data_mapper = BaseDictionaryMapper(
+                    df_input_unit, unit_map_key_columns, output_unit_data_columns
+                )
+                _ = unit_data_mapper.map_data(
+                    df_input_data=df_output_data, input_key_columns=unit_map_key_columns, in_place=True
+                )
+                fillna_with_space(df_output_data)
+        except Exception as e:
+            print(f"Error processing UnitMapper: {e}")
+            raise HTTPException(status_code=500, detail=str(e))
 
+        # AbstractSimilarityMapper
+        try:
+            if sentence_service.df_abstract_map_data is not None:
                 abstract_similarity_mapper = AbstractSimilarityMapper(
                     cached_embedding_helper=sentence_service.abstract_cached_embedding_helper,
                     df_map_data=sentence_service.df_abstract_map_data,
                 )
-                abstract_similarity_mapper.predict_input_optimized(df_input_data=df_output_data)
-
-                print(f"DEBUG: AbstractSimilarityMapper completed successfully")
+                abstract_similarity_mapper.predict_input(df_input_data=df_output_data)
 
         except Exception as e:
             print(f"Error processing AbstractSimilarityMapper: {e}")
             print(f"DEBUG: Full error traceback:")
-            import traceback
-
             traceback.print_exc()
             # Don't raise the exception, continue processing
             print(f"DEBUG: Continuing without AbstractSimilarityMapper...")
 
+        # NameAndAbstractDataMapper
         try:
-            # Name and abstract mapping
             if sentence_service.df_name_and_subject_map_data is not None:
                 name_and_abstract_mapper = NameAndAbstractDataMapper(
                     df_map_data=sentence_service.df_name_and_subject_map_data
                 )
-                df_output_data = name_and_abstract_mapper.map_data(df_output_data)
+                df_output_data["出力_項目名"] = df_output_data["出力_標準名称"]
+                _ = name_and_abstract_mapper.map_data(df_output_data)
+                fillna_with_space(df_output_data)
+                df_output_data["出力_項目名（中科目抜き）"] = df_output_data["出力_項目名"]
 
         except Exception as e:
             print(f"Error processing NameAndAbstractDataMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
+        # SubSubjectLocationDataMapper
         try:
-            # Unit mapping
-            if sentence_service.df_unit_map_data is not None:
-                unit_mapper = UnitSimilarityMapper(
-                    cached_embedding_helper=sentence_service.unit_cached_embedding_helper,
-                    df_map_data=sentence_service.df_unit_map_data,
-                )
-                unit_mapper.predict_input_optimized(df_input_data=df_output_data)
-
-        except Exception as e:
-            print(f"Error processing UnitMapper: {e}")
-            raise HTTPException(status_code=500, detail=str(e))
-
-        try:
-            # Standard name mapping
-            if sentence_service.df_standard_name_map_data is not None:
-                standard_name_mapper = StandardNameMapper(
-                    df_map_data=sentence_service.df_standard_name_map_data
-                )
-                df_output_data = standard_name_mapper.map_data(df_output_data)
-
+            sub_subject_location_mapper = SubSubjectLocationDataMapper()
+            sub_subject_location_mapper.map_location(df_output_data)
+            df_output_data["名称"] = df_output_data["元名称"]
         except Exception as e:
-            print(f"Error processing StandardNameMapper: {e}")
+            print(f"Error processing SubSubjectLocationDataMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
         # Create output columns and ensure they have proper values
@@ -286,7 +296,6 @@ async def predict(
         for col, default_value in required_columns.items():
             if col not in df_output_data.columns:
                 df_output_data[col] = default_value
-
         # Map output columns to match Excel structure
         # 出力_中科目 mapping - use the standard sub-subject from sub-subject mapper
         if "出力_中科目" in df_output_data.columns:
@@ -331,26 +340,26 @@ async def predict(
         print(f"Available columns after processing: {list(df_output_data.columns)}")
 
         # Final check and fallback for missing output columns
-        if (
-            "出力_中科目" not in df_output_data.columns
-            or df_output_data["出力_中科目"].eq("").all()
-        ):
-            df_output_data["出力_中科目"] = df_output_data.get("中科目", "")
-
-        if (
-            "出力_項目名" not in df_output_data.columns
-            or df_output_data["出力_項目名"].eq("").all()
-        ):
-            df_output_data["出力_項目名"] = df_output_data.get("名称", "")
-
-        if (
-            "出力_単位" not in df_output_data.columns
-            or df_output_data["出力_単位"].eq("").all()
-        ):
-            df_output_data["出力_単位"] = df_output_data.get("単位", "")
-
-        if "出力_確率度" not in df_output_data.columns:
-            df_output_data["出力_確率度"] = 0  # Default confidence score
+        # if (
+        #     "出力_中科目" not in df_output_data.columns
+        #     or df_output_data["出力_中科目"].eq("").all()
+        # ):
+        #     df_output_data["出力_中科目"] = df_output_data.get("中科目", "")
+
+        # if (
+        #     "出力_項目名" not in df_output_data.columns
+        #     or df_output_data["出力_項目名"].eq("").all()
+        # ):
+        #     df_output_data["出力_項目名"] = df_output_data.get("名称", "")
+
+        # if (
+        #     "出力_単位" not in df_output_data.columns
+        #     or df_output_data["出力_単位"].eq("").all()
+        # ):
+        #     df_output_data["出力_単位"] = df_output_data.get("単位", "")
+
+        # if "出力_確率度" not in df_output_data.columns:
+        #     df_output_data["出力_確率度"] = 0  # Default confidence score
 
         # Define output columns in exact order as shown in Excel
         output_columns = [
@@ -511,14 +520,14 @@ async def predict_raw(
         try:
             # Unit mapping
             if sentence_service.df_unit_map_data is not None:
-                unit_mapper = UnitSimilarityMapper(
+                unit_mapper = UnitMapper(
                     cached_embedding_helper=sentence_service.unit_cached_embedding_helper,
                     df_map_data=sentence_service.df_unit_map_data,
                 )
                 unit_mapper.predict_input(df_input_data=df_input_data)
 
         except Exception as e:
-            print(f"Error processing UnitSimilarityMapper: {e}")
+            print(f"Error processing UnitMapper: {e}")
             raise HTTPException(status_code=500, detail=str(e))
 
         # Ensure required columns exist

validate_optimization.py

@@ -25,7 +25,7 @@ class FileComparator:
             '出力_中科目', 
             '出力_標準名称', 
             '出力_項目名', 
-            '出力_標準単位'
+            '出力_集計用単位'
         ]
 
     def load_original_data(self) -> pd.DataFrame:
@@ -236,7 +236,7 @@ def main():
     """Main function to compare two files"""
     # File paths
     original_file = "data/outputData_original.csv"
-    second_file = "data/outputData_api_v2.csv"
+    second_file = "data/outputData_api.csv"
 
     if not os.path.exists(original_file):
         print(f"❌ Original file not found: {original_file}")