优化预测函数的输入文本打印逻辑，增加文本长度信息；改进长文本处理函数，考虑特殊标记长度以保持句子完整性

app.py

@@ -52,8 +52,8 @@ async def predict(request: PredictRequest):
     try:
         input_text = request.text  # FastAPI 会自动解析为 PredictRequest 对象
         affected_stock_codes = request.stock_codes
-        print("Input text:", input_text[:200] if len(input_text) > 200 else input_text)
-        print("Affected stock codes:", affected_stock_codes)
+        print(f"Input Text Length: {len(input_text)}, Start with: {input_text[:200] if len(input_text) > 200 else input_text}")
+        print("Input stock codes:", affected_stock_codes)
         return predict(input_text, affected_stock_codes)
     except Exception as e:
         return {"error": str(e)}

preprocess.py

@@ -10,6 +10,7 @@ import pandas as pd
 import time
 
 # 如果使用 spaCy 进行 NLP 处理
+from regex import R
 import spacy
 
 # 如果使用某种情感分析工具，比如 Hugging Face 的模型
@@ -225,7 +226,7 @@ def get_document_vector(words, model = word2vec_model):
 # 函数：获取情感得分
 def process_long_text(text, tokenizer, max_length=512):
     """
-    将长文本分段并保持句子完整性
+    将长文本分段并保持句子完整性，同时考虑特殊标记的长度
     """
     import nltk
     try:
@@ -239,15 +240,19 @@ def process_long_text(text, tokenizer, max_length=512):
         nltk.download('punkt_tab')
         
         
+    # 计算特殊标记占用的长度(CLS, SEP等)
+    special_tokens_count = tokenizer.num_special_tokens_to_add()
+    # 实际可用于文本的最大长度
+    effective_max_length = max_length - special_tokens_count
+    
     sentences = nltk.sent_tokenize(text)
     segments = []
     current_segment = ""
     
     for sentence in sentences:
-        print(f"Processing sentence: {sentence}")
         # 检查添加当前句子后是否会超过最大长度
         test_segment = current_segment + " " + sentence if current_segment else sentence
-        if len(tokenizer.tokenize(test_segment)) > max_length:
+        if len(tokenizer.tokenize(test_segment)) > effective_max_length:
             if current_segment:
                 segments.append(current_segment.strip())
             current_segment = sentence
@@ -340,6 +345,7 @@ def get_sentiment_score(text):
         return 0.0
 
 
+
 def get_stock_info(stock_code: str, history_days=30):
     # 获取股票代码和新闻日期