language_checker.py

# language_checker.py
import re
import traceback
from typing import List, Dict, Any
import language_tool_python
import logging # For more persistent error messages

from text_utils import convert_markdown_to_plain_text
# config.py (setting JAVA_HOME) should be imported early in app.py

# Import SpanMarkerModel
try:
    from span_marker import SpanMarkerModel
    SPAN_MARKER_AVAILABLE = True
except ImportError:
    SPAN_MARKER_AVAILABLE = False
    SpanMarkerModel = None # Placeholder if not available
    print("LT_Checker: Warning: span_marker library not found. Acronym filtering will be disabled.")
    print("LT_Checker: Please install it via 'pip install span_marker'")


# --- Global SpanMarker Model for Acronyms ---
_span_marker_model_acronyms = None
_span_marker_model_loaded_successfully = False
_span_marker_model_load_attempted = False

SPAN_MARKER_ACRONYM_MODEL_NAME = "tomaarsen/span-marker-bert-base-uncased-acronyms"

def _load_span_marker_model_if_needed():
    global _span_marker_model_acronyms, _span_marker_model_loaded_successfully, _span_marker_model_load_attempted

    if not SPAN_MARKER_AVAILABLE or _span_marker_model_load_attempted:
        return

    _span_marker_model_load_attempted = True
    try:
        print(f"LT_Checker: Attempting to load SpanMarker model '{SPAN_MARKER_ACRONYM_MODEL_NAME}' for acronym detection...")
        # Ensure you have torch installed, or the appropriate backend for SpanMarkerModel
        _span_marker_model_acronyms = SpanMarkerModel.from_pretrained(SPAN_MARKER_ACRONYM_MODEL_NAME)
        _span_marker_model_loaded_successfully = True
        print(f"LT_Checker: SpanMarker model '{SPAN_MARKER_ACRONYM_MODEL_NAME}' loaded successfully.")
    except Exception as e:
        _span_marker_model_loaded_successfully = False
        print(f"LT_Checker: CRITICAL ERROR loading SpanMarker model '{SPAN_MARKER_ACRONYM_MODEL_NAME}': {e}")
        print(f"LT_Checker: Acronym filtering will be disabled. Please check your installation and model availability.")
        logging.error(f"Failed to load SpanMarker model '{SPAN_MARKER_ACRONYM_MODEL_NAME}': {e}", exc_info=True)

# Attempt to load the model when the module is first imported.
# This might slightly delay the initial import if the model is large.
_load_span_marker_model_if_needed()


def _is_text_acronym_related(text_to_check: str, acronym_entities: List[Dict[str, Any]]) -> bool:
    """
    Checks if the text_to_check contains any of the acronyms (long or short form)
    identified by the SpanMarker model.
    """
    if not acronym_entities or not text_to_check:
        return False
    
    text_to_check_lower = text_to_check.lower()
    for entity in acronym_entities:
        acronym_span = entity.get('span', '')
        if acronym_span: # Ensure span is not empty
            # Check if the identified acronym span is present in the text flagged by LanguageTool
            if acronym_span.lower() in text_to_check_lower:
                # print(f"Debug AcronymFilter: Text '{text_to_check}' (from LT) contains detected acronym '{acronym_span}'. Filtering.")
                return True
    return False


def perform_language_checks(markdown_text_from_filtered_pdf: str) -> Dict[str, Any]:
    """
    Performs LanguageTool checks on plain text derived from font-filtered Markdown.
    Filters issues to only include those between "abstract" and "references/bibliography"
    found within this specific text.
    Also filters out issues related to acronyms identified by SpanMarker.
    """
    if not markdown_text_from_filtered_pdf or not markdown_text_from_filtered_pdf.strip():
        print("LT_Checker: Input Markdown text is empty.")
        return {"total_issues": 0, "issues_list": [], "text_used_for_analysis": ""}

    plain_text_from_markdown = convert_markdown_to_plain_text(markdown_text_from_filtered_pdf)
    text_for_lt_analysis = plain_text_from_markdown.replace('\n', ' ')
    text_for_lt_analysis = re.sub(r'\s+', ' ', text_for_lt_analysis).strip()

    if not text_for_lt_analysis:
        print("LT_Checker: Plain text derived from Markdown is empty after cleaning.")
        return {"total_issues": 0, "issues_list": [], "text_used_for_analysis": ""}

    # --- Acronym Detection using SpanMarker ---
    acronym_entities = []
    if _span_marker_model_loaded_successfully and _span_marker_model_acronyms:
        try:
            # print(f"LT_Checker: Running SpanMarker on text of length {len(text_for_lt_analysis)} for acronyms.")
            acronym_entities = _span_marker_model_acronyms.predict(text_for_lt_analysis)
            # if acronym_entities:
            #     print(f"LT_Checker: SpanMarker found {len(acronym_entities)} acronym entities. Examples: {[e['span'] for e in acronym_entities[:3]]}")
        except Exception as sm_e:
            print(f"LT_Checker: Error during SpanMarker prediction: {sm_e}")
            logging.warning(f"SpanMarker prediction failed: {sm_e}", exc_info=True)
            # Proceed without acronym filtering if prediction fails
            acronym_entities = []
    elif SPAN_MARKER_AVAILABLE and not _span_marker_model_loaded_successfully:
        print("LT_Checker: SpanMarker model was available but not loaded successfully. Acronym filtering disabled for this run.")


    text_for_lt_analysis_lower = text_for_lt_analysis.lower()
    
    abstract_match = re.search(r'\babstract\b', text_for_lt_analysis_lower)
    content_start_index = abstract_match.start() if abstract_match else 0
    # ... (rest of abstract/references boundary logic as before) ...
    if abstract_match:
        print(f"LT_Checker: Found 'abstract' at index {content_start_index} in its text.")
    else:
        print(f"LT_Checker: Did not find 'abstract', LT analysis from index 0 of its text.")

    references_match = re.search(r'\breferences\b', text_for_lt_analysis_lower)
    bibliography_match = re.search(r'\bbibliography\b', text_for_lt_analysis_lower)
    content_end_index = len(text_for_lt_analysis)

    if references_match and bibliography_match:
        content_end_index = min(references_match.start(), bibliography_match.start())
        print(f"LT_Checker: Found 'references' at {references_match.start()} and 'bibliography' at {bibliography_match.start()}. Using {content_end_index} as end boundary.")
    elif references_match:
        content_end_index = references_match.start()
        print(f"LT_Checker: Found 'references' at {content_end_index}. Using it as end boundary.")
    elif bibliography_match:
        content_end_index = bibliography_match.start()
        print(f"LT_Checker: Found 'bibliography' at {content_end_index}. Using it as end boundary.")
    else:
        print(f"LT_Checker: Did not find 'references' or 'bibliography'. LT analysis up to end of its text (index {content_end_index}).")

    if content_start_index >= content_end_index:
        print(f"LT_Checker: Warning: Content start index ({content_start_index}) is not before end index ({content_end_index}) in its text. No LT issues will be reported from this range.")
    
    tool = None
    processed_lt_issues: List[Dict[str, Any]] = []
    try:
        tool = language_tool_python.LanguageTool('en-US') 
        raw_lt_matches = tool.check(text_for_lt_analysis)
        
        lt_issues_in_range = 0
        filtered_acronym_issues = 0

        for idx, match in enumerate(raw_lt_matches):
            if match.ruleId == "EN_SPLIT_WORDS_HYPHEN": continue  # Common rule to ignore

            # --- Acronym Filtering Step ---
            if acronym_entities and _is_text_acronym_related(match.matchedText, acronym_entities):
                filtered_acronym_issues += 1
                continue # Skip this LanguageTool match as it's related to a detected acronym

            if not (content_start_index <= match.offset < content_end_index):
                continue
            lt_issues_in_range += 1

            error_text_verbatim = match.matchedText 
            words_around = 1
            pre_error_text = text_for_lt_analysis[:match.offset]
            words_before = pre_error_text.split()[-words_around:]
            post_error_text = text_for_lt_analysis[match.offset + match.errorLength:]
            words_after = post_error_text.split()[:words_around]
            context_parts = []
            if words_before: context_parts.append(" ".join(words_before))
            context_parts.append(error_text_verbatim)
            if words_after: context_parts.append(" ".join(words_after))
            wider_context_str = " ".join(context_parts)

            processed_lt_issues.append({
                '_internal_id': f"lt_{idx}",
                'ruleId': match.ruleId,
                'message': match.message,
                'context_text': wider_context_str,
                'error_text_verbatim': error_text_verbatim,
                'offset_in_text': match.offset,
                'error_length': match.errorLength,
                'replacements_suggestion': match.replacements[:3] if match.replacements else [],
                'category_name': match.category,
                'source_check_type': 'LanguageTool',
                'is_mapped_to_pdf': False,
                'pdf_coordinates_list': [],
                'mapped_page_number': -1
            })
        
        print(f"LT_Checker: LanguageTool found {len(raw_lt_matches)} raw issues.")
        if acronym_entities:
            print(f"LT_Checker: Filtered out {filtered_acronym_issues} LT issues due to acronym detection.")
        print(f"LT_Checker: {lt_issues_in_range} LT issues within defined content range (after acronym filtering).")
        
        return {
            "total_issues": len(processed_lt_issues),
            "issues_list": processed_lt_issues,
            "text_used_for_analysis": text_for_lt_analysis 
        }
    except Exception as e:
        print(f"Error in perform_language_checks: {e}\n{traceback.format_exc()}")
        return {"error": str(e), "total_issues": 0, "issues_list": [], "text_used_for_analysis": text_for_lt_analysis}
    finally:
        if tool:
            tool.close()