Merge pull request #13 from Sistem-Cerdas-Recruitment/production

core-model-prediction/prediction.py

@@ -2,7 +2,7 @@ from fastapi import FastAPI, Response, status
 from pydantic import BaseModel
 from hypothesis import BaseModelHypothesis
 from secondary_model_dependencies import SecondaryModelDependencies
-from random_forest_model import RandomForestModel
+from secondary_model import SecondaryModel
 from main_model import PredictMainModel
 import numpy as np
 from typing import List
@@ -11,13 +11,10 @@ app = FastAPI()
 
 
 class PredictRequest(BaseModel):
-    question: str
     answer: str
     backspace_count: int
-    typing_duration: int
     letter_click_counts: dict[str, int]
-    gpt35_answer: str
-    gpt4_answer: str
+    gpt4o_answer: str
 
 
 class RequestModel(BaseModel):
@@ -36,13 +33,10 @@ async def predict(request: RequestModel):
 
 
 def process_instance(data: PredictRequest):
-    question = data.question
     answer = data.answer
     backspace_count = data.backspace_count
-    typing_duration = data.typing_duration
     letter_click_counts = data.letter_click_counts
-    gpt35_answer = data.gpt35_answer
-    gpt4_answer = data.gpt4_answer
+    gpt4o_answer = data.gpt4o_answer
 
     # Data preparation for 1st model
     hypothesis = BaseModelHypothesis()
@@ -56,26 +50,28 @@ def process_instance(data: PredictRequest):
     # Data preparation for 2nd model
     secondary_model_dependencies = SecondaryModelDependencies()
     secondary_model_features = secondary_model_dependencies.calculate_features(
-        question, answer, main_model_probability, backspace_count, typing_duration,
-        letter_click_counts, gpt35_answer, gpt4_answer)
+        answer, main_model_probability, backspace_count,
+        letter_click_counts, gpt4o_answer)
 
     # 2nd model prediction
-    secondary_model = RandomForestModel()
-    secondary_model_prediction = secondary_model.predict(
+    secondary_model = SecondaryModel()
+    secondary_model_probability = secondary_model.predict(
         secondary_model_features)
 
+    second_model_threshold = 0.54
+
     return {
-        "predicted_class": "AI" if secondary_model_prediction == 1 else "HUMAN",
+        "predicted_class": "AI" if secondary_model_probability > second_model_threshold else "HUMAN",
         "main_model_probability": str(main_model_probability),
-        "secondary_model_prediction": secondary_model_prediction,
-        "confidence": get_confidence(main_model_probability, secondary_model_prediction)
+        "secondary_model_probability": str(secondary_model_probability),
+        "confidence": get_confidence(main_model_probability, secondary_model_probability, second_model_threshold)
     }
 
 
-def get_confidence(main_model_output: float, secondary_model_output: int):
-    if (main_model_output >= 0.8 and secondary_model_output == 1) or (main_model_output <= 0.2 and secondary_model_output == 0):
+def get_confidence(main_model_output: float, secondary_model_output: int, threshold: float):
+    if (main_model_output >= 0.8 and secondary_model_output >= threshold) or (main_model_output <= 0.2 and secondary_model_output <= 1 - threshold):
         return 'High Confidence'
-    elif (0.5 < main_model_output < 0.8 and secondary_model_output == 1) or (0.2 < main_model_output <= 0.5 and secondary_model_output == 0):
+    elif (0.5 < main_model_output < 0.8 and secondary_model_output >= threshold) or (0.2 < main_model_output <= 0.5 and secondary_model_output < threshold):
         return 'Partially Confident'
     else:
         return 'Low Confidence'

core-model-prediction/{random_forest_model.py → secondary_model.py}

@@ -4,21 +4,21 @@ import pandas as pd
 from typing import List
 
 
-class RandomForestModel:
+class SecondaryModel:
     def __init__(self):
         self.scaler = joblib.load("scalers/secondary_scaler.joblib")
-        self.model = joblib.load("models/rf_weights.joblib")
+        self.model = joblib.load("models/secondary_weights.joblib")
         self.secondary_model_features = [
-            "machine_probability", "backspace_count_normalized", "typing_duration_normalized",
-            "letter_discrepancy_normalized", "cosine_sim_gpt35", "cosine_sim_gpt4"
+            "machine_probability", "backspace_count_normalized",
+            "letter_discrepancy_normalized", "cosine_sim_gpt4o"
         ]
 
-    def preprocess_input(self, secondary_model_features: List[float]) -> np.ndarray:
+    def preprocess_input(self, secondary_model_features: List[float]) -> pd.DataFrame:
         features_df = pd.DataFrame(
             [secondary_model_features], columns=self.secondary_model_features)
         features_df[self.secondary_model_features] = self.scaler.transform(
             features_df[self.secondary_model_features])
-        return features_df.values.astype(np.float32).reshape(1, -1)
+        return features_df
 
-    def predict(self, secondary_model_features: List[float]):
-        return int(self.model.predict(self.preprocess_input(secondary_model_features))[0])
+    def predict(self, secondary_model_features: List[float]) -> float:
+        return self.model.predict_proba(self.preprocess_input(secondary_model_features))[:, -1][0]

core-model-prediction/secondary_model_dependencies.py

@@ -7,20 +7,16 @@ class SecondaryModelDependencies:
         self.text_similarity_model = SentenceTransformer(
             'sentence-transformers/all-mpnet-base-v2')
 
-    def calculate_features(self, question: str, answer: str, probability: float, backspace_count: int, typing_duration: int,
-                           letter_click_counts: dict[str, int], gpt35_answer: str, gpt4_answer: str):
+    def calculate_features(self, answer: str, probability: float, backspace_count: int,
+                           letter_click_counts: dict[str, int], gpt4o_answer: str):
         backspace_count_normalized = backspace_count / len(answer)
-        typing_duration_normalized = typing_duration / len(answer)
         letter_discrepancy = self.calculate_letter_discrepancy(
             answer, letter_click_counts)
-
-        cosine_sim_gpt35 = self.calculate_similarity_gpt35(
-            answer, gpt35_answer)
-        cosine_sim_gpt4 = self.calculate_similarity_gpt4(answer, gpt4_answer)
+        cosine_sim_gpt4o = self.calculate_similarity_gpt4o(
+            answer, gpt4o_answer)
 
         return [
-            probability, backspace_count_normalized, typing_duration_normalized,
-            letter_discrepancy, cosine_sim_gpt35, cosine_sim_gpt4
+            probability, backspace_count_normalized, letter_discrepancy, cosine_sim_gpt4o
         ]
 
     def calculate_letter_discrepancy(self, text: str, letter_click_counts: dict[str, int]):
@@ -38,18 +34,10 @@ class SecondaryModelDependencies:
 
         return discrepancy_ratio_normalized
 
-    def calculate_similarity_gpt35(self, answer: str, gpt35_answer: str) -> float:
-        embedding1 = self.text_similarity_model.encode(
-            [answer], convert_to_tensor=True)
-        embedding2 = self.text_similarity_model.encode(
-            [gpt35_answer], convert_to_tensor=True)
-        cosine_scores = util.cos_sim(embedding1, embedding2)
-        return cosine_scores.item()
-
-    def calculate_similarity_gpt4(self, answer: str, gpt4_answer: str) -> float:
+    def calculate_similarity_gpt4o(self, answer: str, gpt4o_answer: str) -> float:
         embedding1 = self.text_similarity_model.encode(
             [answer], convert_to_tensor=True)
         embedding2 = self.text_similarity_model.encode(
-            [gpt4_answer], convert_to_tensor=True)
+            [gpt4o_answer], convert_to_tensor=True)
         cosine_scores = util.cos_sim(embedding1, embedding2)
         return cosine_scores.item()