Enhance prediction process by adding missing columns with defaults, ensuring correct data types, and improving error handling. Update cached embeddings with new size.

data/cached_embeddings_unit.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:244d9e2e89d023dfcd8eb8eddf81d4295b3028e2ebb19b01638af08432edb6c8
-size 730951
+oid sha256:917d6d46ef5e75ddca3f081169eb9f9323eab50dbed95583037907c26c855ae0
+size 734106

routes/predict.py

@@ -174,15 +174,35 @@ async def predict(
         try:
             # Abstract mapping
             if sentence_service.df_abstract_map_data is not None:
-                # Ensure required columns exist for AbstractSimilarityMapper
+                # Ensure required columns exist before AbstractSimilarityMapper
                 required_columns_for_abstract = {
+                    "標準科目": "",
                     "摘要グループ": "",
-                    "確定": "",
+                    "確定": "未確定",
+                    "摘要": "",
+                    "備考": "",
                 }
 
-                for col, default_value in required_columns_for_abstract.items():
+                # Add missing columns with appropriate defaults
+                for col, default_val in required_columns_for_abstract.items():
                     if col not in df_output_data.columns:
-                        df_output_data[col] = default_value
+                        df_output_data[col] = default_val
+                        print(
+                            f"DEBUG: Added missing column '{col}' with default value '{default_val}'"
+                        )
+
+                # Ensure data types are correct (convert to string to avoid type issues)
+                for col in ["標準科目", "摘要グループ", "確定", "摘要", "備考"]:
+                    if col in df_output_data.columns:
+                        df_output_data[col] = df_output_data[col].astype(str).fillna("")
+
+                # Debug: Print sample data before AbstractSimilarityMapper
+                print(f"DEBUG: Sample data before AbstractSimilarityMapper:")
+                print(
+                    df_output_data[["標準科目", "摘要グループ", "確定", "摘要", "備考"]]
+                    .head(3)
+                    .to_string()
+                )
 
                 abstract_similarity_mapper = AbstractSimilarityMapper(
                     cached_embedding_helper=sentence_service.abstract_cached_embedding_helper,
@@ -190,9 +210,16 @@ async def predict(
                 )
                 abstract_similarity_mapper.predict_input(df_input_data=df_output_data)
 
+                print(f"DEBUG: AbstractSimilarityMapper completed successfully")
+
         except Exception as e:
             print(f"Error processing AbstractSimilarityMapper: {e}")
-            raise HTTPException(status_code=500, detail=str(e))
+            print(f"DEBUG: Full error traceback:")
+            import traceback
+
+            traceback.print_exc()
+            # Don't raise the exception, continue processing
+            print(f"DEBUG: Continuing without AbstractSimilarityMapper...")
 
         try:
             # Name and abstract mapping
@@ -290,25 +317,6 @@ async def predict(
         # Fill NaN values and ensure all output columns have proper values
         df_output_data = df_output_data.fillna("")
 
-        # Convert columns to string to avoid dtype issues
-        string_columns = [
-            "摘要グループ",
-            "確定",
-            "出力_基準中科目",
-            "出力_中科目",
-            "出力_項目名",
-            "出力_標準名称",
-            "出力_基準名称",
-            "出力_単位",
-            "出力_集計用単位",
-            "出力_標準単位",
-            "出力_基準単位",
-            "外部・内部区分",
-        ]
-        for col in string_columns:
-            if col in df_output_data.columns:
-                df_output_data[col] = df_output_data[col].astype(str).replace("nan", "")
-
         # Debug: Print available columns to see what we have
         print(f"Available columns after processing: {list(df_output_data.columns)}")
 

services/sentence_transformer_service.py

@@ -211,6 +211,13 @@ class SentenceTransformerService:
                 print(
                     f"Loaded abstract map data: {len(self.df_abstract_map_data)} entries"
                 )
+                print(
+                    f"DEBUG: Abstract map data columns: {list(self.df_abstract_map_data.columns)}"
+                )
+                print(f"DEBUG: Abstract map data sample:")
+                print(self.df_abstract_map_data.head(3).to_string())
+            else:
+                print(f"DEBUG: Abstract map file not found: {abstract_map_file}")
 
             # Load name and subject map data
             name_and_subject_map_file = os.path.join(