Initial commit of Paragraph_Checker module for Hugging Face

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.10-slim
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    openjdk-17-jre-headless \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set up application
+WORKDIR /app
+COPY . .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Pre-download models during build
+RUN python -c "from paragraph_checker import initialize_models; initialize_models()"
+
+EXPOSE 5001
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,42 @@
+from flask import Flask, request, jsonify
+from paragraph_checker import correct_paragraph
+
+app = Flask(__name__)
+
+@app.route('/correct_text', methods=['POST'])
+def correct_text():
+    data = request.get_json()
+    text = data.get("paragraph", "")
+
+    if not text:
+        return jsonify({"error": "No text provided"}), 400
+
+    print("Original Text:", text)
+
+    try:
+        # Get fully corrected text
+        fully_corrected = correct_paragraph(text)
+
+        return jsonify({
+            "original_text": text,
+            "corrected_text": fully_corrected
+        })
+
+    except Exception as e:
+        print(f"Error processing text: {str(e)}")
+        return jsonify({
+            "error": "An error occurred while processing the text",
+            "details": str(e)
+        }), 500
+
+if __name__ == '__main__':
+    # Initialize models at startup
+    from paragraph_checker import initialize_models
+    print("Loading ML models...")
+    try:
+        initialize_models()
+        print("Models loaded successfully")
+    except Exception as e:
+        print(f"Error loading models: {str(e)}")
+
+    app.run(host="0.0.0.0", port=5001, debug=True)

paragraph_checker.py

@@ -0,0 +1,67 @@
+import language_tool_python
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+
+# Global variables for models
+grammar_tool = None
+tense_model = None
+tense_tokenizer = None
+
+def initialize_models():
+    """Initialize all ML models at startup"""
+    global grammar_tool, tense_model, tense_tokenizer
+
+    print("Initializing Language Tool...")
+    grammar_tool = language_tool_python.LanguageTool('en-US')
+
+    print("Initializing T5 model...")
+    model_name = "Vamsi/T5_Paraphrase_Paws"
+    tense_tokenizer = T5Tokenizer.from_pretrained(model_name)
+    tense_model = T5ForConditionalGeneration.from_pretrained(model_name)
+
+def grammar_correction(text):
+    """Correct grammar using LanguageTool"""
+    if not grammar_tool:
+        raise Exception("Grammar tool not initialized")
+
+    matches = grammar_tool.check(text)
+    corrected_text = language_tool_python.utils.correct(text, matches)
+    return corrected_text
+
+def tense_correction(text):
+    """Correct tense using T5 model"""
+    if not tense_model or not tense_tokenizer:
+        raise Exception("Tense correction models not initialized")
+
+    input_text = "paraphrase: " + text + " </s>"
+    encoding = tense_tokenizer.encode_plus(
+        input_text,
+        padding='max_length',
+        return_tensors="pt",
+        max_length=256,
+        truncation=True
+    )
+    input_ids, attention_masks = encoding["input_ids"], encoding["attention_mask"]
+
+    outputs = tense_model.generate(
+        input_ids=input_ids,
+        attention_mask=attention_masks,
+        max_length=256,
+        num_return_sequences=1,
+        num_beams=5,
+        temperature=1.5
+    )
+
+    paraphrased = tense_tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return paraphrased
+
+def correct_paragraph(text):
+    """Complete text correction pipeline"""
+    # Step 1: Grammar correction
+    grammatically_correct = grammar_correction(text)
+    print("After Grammar Correction:", grammatically_correct)
+
+    # Step 2: Tense correction
+    fully_corrected = tense_correction(grammatically_correct)
+    print("After Grammar + Tense Correction:", fully_corrected)
+
+    return fully_corrected

requirements.txt

@@ -0,0 +1,6 @@
+flask
+language-tool-python
+transformers
+torch
+sentencepiece
+protobuf