generate_multilingual_bib.py

#!/usr/bin/env python3
"""
Multilingual Paper BibTeX Generator

This script parses anthology+abstracts.bib and generates multilingual_papers.bib
containing only papers related to multilingual NLP research.

Usage:
    python generate_multilingual_bib.py

Requirements:
    - anthology+abstracts.bib file in the same directory
"""

import re
import os
from typing import List, Dict, Set
from tqdm import tqdm
from collections import defaultdict

# Multilingual keywords for filtering (same as JavaScript version)
MULTILINGUAL_KEYWORDS = [
    'multilingual', 'crosslingual', 'multi lingual', 'cross lingual', 
    'multi-lingual', 'cross-lingual', 'low-resource language', 'low resource language', 
    # 'low-resource', 'low resource',
    'multi-language', 'multi language', 'cross-language', 'cross language',
    'language transfer', 
    'code-switching', 'code switching', 'language adaptation',
    'language pair', 'bilingual', 'trilingual', 'polyglot',
    # 'machine translation', 'neural machine translation', 'speech translation', 'translation', 'nmt',
    'translation', "nmt",
    'transliteration',
    'multilingual bert', 'xlm', 'mbert', 'xlm-roberta',
    'language identification', 'language detection'
]

# Language names for filtering (same as JavaScript version)
LANGUAGE_NAMES = [
    'afrikaans', 'albanian', 'amharic', 'arabic', 'armenian', 'azerbaijani', 'basque', 'belarusian', 'bengali', 'bosnian', 'bulgarian', 'catalan', 'cebuano', 'chinese', 'croatian', 'czech', 'danish', 'dutch', 'esperanto', 'estonian', 'filipino', 'finnish', 'french', 'galician', 'georgian', 'german', 'greek', 'gujarati', 'haitian', 'hausa', 'hawaiian', 'hebrew', 'hindi', 'hmong', 'hungarian', 'icelandic', 'igbo', 'indonesian', 'irish', 'italian', 'japanese', 'javanese', 'kannada', 'kazakh', 'khmer', 'korean', 'kurdish', 'kyrgyz', 'lao', 'latin', 'latvian', 'lithuanian', 'luxembourgish', 'macedonian', 'malagasy', 'malay', 'malayalam', 'maltese', 'maori', 'marathi', 'mongolian', 'myanmar', 'nepali', 'norwegian', 'odia', 'pashto', 'persian', 'polish', 'portuguese', 'punjabi', 'romanian', 'russian', 'samoan', 'scots gaelic', 'serbian', 'sesotho', 'shona', 'sindhi', 'sinhala', 'slovak', 'slovenian', 'somali', 'spanish', 'sundanese', 'swahili', 'swedish', 'tagalog', 'tajik', 'tamil', 'telugu', 'thai', 'turkish', 'ukrainian', 'urdu', 'uzbek', 'vietnamese', 'welsh', 'xhosa', 'yiddish', 'yoruba', 'zulu',
    # Additional language variations and names
    'mandarin', 'cantonese', 'hindi', 'urdu', 'bengali', 'tamil', 'telugu', 'marathi', 'gujarati', 'kannada', 'malayalam', 'punjabi', 'odia', 'assamese', 'maithili', 'sanskrit', 'kashmiri', 'konkani', 'manipuri', 'nepali', 'sindhi', 'dogri', 'bodo', 'santali', 'khasi', 'mizo', 'garo', 'naga', 'tibetan', 'dzongkha', 'sikkimese', 'lepcha', 'limbu', 'tamang', 'gurung', 'magar', 'tharu', 'tulu'
    'african', 'indian', "asian", "indigenous",
]

def clean_latex_commands(text: str) -> str:
    """
    Clean LaTeX commands from text (same logic as JavaScript version).
    """
    if not text:
        return ''
    
    # Remove LaTeX commands with braces
    text = re.sub(r'\\[a-zA-Z]+\{([^}]*)\}', r'\1', text)
    # Remove simple LaTeX commands
    text = re.sub(r'\\[a-zA-Z]+', '', text)
    # Remove braces
    text = re.sub(r'\{\\?([^}]*)\}', r'\1', text)
    # Replace escaped characters
    text = text.replace('\\"', '"')
    text = text.replace("\\'", "'")
    text = text.replace('\\&', '&')
    text = text.replace('\\%', '%')
    text = text.replace('\\_', '_')
    text = text.replace('\\$', '$')
    # Normalize whitespace
    text = re.sub(r'\s+', ' ', text)
    
    return text.strip()

def is_multilingual_paper(paper: Dict[str, str]) -> bool:
    """
    Determine if a paper is multilingual (same logic as JavaScript version).
    """
    text = f"{paper.get('title', '')} {paper.get('abstract', '')}".lower()
    
    # Check for multilingual keywords
    for keyword in MULTILINGUAL_KEYWORDS:
        if keyword.lower() in text:
            return True, keyword

    # Check for language names
    for language in LANGUAGE_NAMES:
        # require language to be matched perfectly
        if language.lower() in text.split():
            return True, language

    return False, None

def extract_keywords(paper: Dict[str, str]) -> Set[str]:
    """
    Extract multilingual keywords from a paper (same logic as JavaScript version).
    """
    keywords = set()
    text = f"{paper.get('title', '')} {paper.get('abstract', '')}".lower()
    
    # Extract multilingual keywords
    for keyword in MULTILINGUAL_KEYWORDS:
        if keyword.lower() in text:
            keywords.add(keyword)
    
    # Extract language names
    for language in LANGUAGE_NAMES:
        if language.lower() in text:
            keywords.add(language)
    
    return keywords

def parse_bibtex_entry(entry: str) -> Dict[str, str]:
    """
    Parse a single BibTeX entry (same logic as JavaScript version).
    """
    paper = {}
    
    # Extract entry type and key
    type_match = re.match(r'@(\w+)\{([^,]+)', entry)
    if type_match:
        paper['type'] = type_match.group(1)
        paper['key'] = type_match.group(2)
    else:
        # If we can't parse the type/key, skip this entry
        return None

    # Extract fields
    fields = ['title', 'author', 'abstract', 'year', 'booktitle', 'journal', 'pages']
    
    for field in fields:
        # Match both {content} and "content" formats
        pattern = rf'{field}\s*=\s*{{([^}}]*)}}|{field}\s*=\s*"([^"]*)"'
        match = re.search(pattern, entry, re.IGNORECASE)
        if match:
            value = match.group(1) or match.group(2)
            # Clean up LaTeX commands
            value = clean_latex_commands(value)
            paper[field] = value.strip()
    
    # Extract year from key if not found in fields
    if not paper.get('year') and paper.get('key'):
        year_match = re.search(r'\d{4}', paper['key'])
        if year_match:
            paper['year'] = year_match.group(0)
    
    # Determine if paper is multilingual
    paper['is_multilingual'], matched_keyword = is_multilingual_paper(paper)
    paper['keywords'] = list(extract_keywords(paper))
    paper['matched_keyword'] = matched_keyword

    return paper

def parse_bibtex(bib_text: str) -> List[Dict[str, str]]:
    """
    Parse BibTeX text into list of paper dictionaries (same logic as JavaScript version).
    """
    papers = []
    # Split by @ to get individual entries
    entries = re.split(r'(?=@)', bib_text)
    n_missing, n_total = 0, len(entries)

    for entry in tqdm(entries, desc="Parsing BibTeX entries"):
        if not entry.strip():
            continue

        # try:
        paper = parse_bibtex_entry(entry)
        if paper and (paper.get('title') or paper.get('abstract')):
            papers.append(paper)
        elif paper is None:
            n_missing += 1
            # print(f"Warning: Skipping malformed entry (no type/key found)")
        # except Exception as e:
            # print(f"Warning: Error parsing entry: {e}")
            continue

    keyword2count = defaultdict(int)
    for paper in papers:
        if not paper['matched_keyword']: continue
        keyword2count[paper['matched_keyword']] += 1
    n_multilingual_papers = sum(keyword2count.values())

    print(f"Found {len(papers)} papers out of {n_total} total papers. Ratio: {len(papers)/n_total*100:.1f}%")
    print(f"Missing {n_missing} papers out of {n_total} total papers. Ratio: {n_missing/n_total*100:.1f}%")

    # sort by keyword count
    keyword2count = sorted(keyword2count.items(), key=lambda x: x[1], reverse=True)
    for keyword, count in keyword2count:
        print(f"\t {keyword}: {count} papers ({count/n_multilingual_papers*100:.1f}%)")

    return papers

def generate_bibtex_content(papers: List[Dict[str, str]]) -> str:
    """
    Generate BibTeX content from paper dictionaries (same logic as JavaScript version).
    """
    content = ''

    for paper in tqdm(papers, desc="Generating BibTeX content"):
        # Check if paper has required fields
        if not paper.get('type') or not paper.get('key'):
            print(f"Warning: Skipping paper without type or key: {paper.get('title', 'Unknown')[:50]}...")
            continue
            
        # Reconstruct the original BibTeX entry
        content += f"@{paper['type']}{{{paper['key']},\n"
        
        fields = ['title', 'author', 'abstract', 'year', 'booktitle', 'journal', 'pages']
        for field in fields:
            if paper.get(field):
                content += f"  {field} = {{{paper[field]}}},\n"
        
        # Remove trailing comma and add closing brace
        content = content.rstrip(',\n') + '\n'
        content += '}\n\n'
    
    return content

def main():
    """
    Main function to generate multilingual_papers.bib.
    """
    input_file = 'data/anthology+abstracts.bib'
    output_file = 'data/multilingual_papers.bib'
    
    # Check if input file exists
    if not os.path.exists(input_file):
        print(f"Error: {input_file} not found in current directory.")
        print("Please ensure the file exists and run the script again.")
        return
    
    # Check if output file already exists
    if os.path.exists(output_file):
        print(f"Warning: {output_file} already exists.")
        response = input("Do you want to overwrite it? (y/N): ")
        if response.lower() != 'y':
            print("Operation cancelled.")
            return
    
    print(f"Reading {input_file}...")
    
    # try:
    if True:
        # Read the input file
        with open(input_file, 'r', encoding='utf-8') as f:
            bib_text = f.read()
        
        all_papers = parse_bibtex(bib_text)
        print(f"Found {len(all_papers)} total papers")

        # Filter multilingual papers
        multilingual_papers = [paper for paper in all_papers if paper['is_multilingual']]
        print(f"Found {len(multilingual_papers)} multilingual papers out of {len(all_papers)} total papers. Ratio: {len(multilingual_papers)/len(all_papers)*100:.1f}%")

        if not multilingual_papers:
            print("No multilingual papers found. Check your keywords and language lists.")
            return

        # Generate BibTeX content
        print("Generating BibTeX content...")
        bib_content = generate_bibtex_content(multilingual_papers)
        
        # Write to output file
        print(f"Writing to {output_file}...")
        with open(output_file, 'w', encoding='utf-8') as f:
            f.write(bib_content)
        
        print(f"Successfully generated {output_file} with {len(multilingual_papers)} papers!")
        
        # Show some statistics
        print("\nStatistics:")
        print(f"  Total papers processed: {len(all_papers)}")
        print(f"  Multilingual papers found: {len(multilingual_papers)}")
        print(f"  Percentage multilingual: {len(multilingual_papers)/len(all_papers)*100:.1f}%")
        
        # Show top keywords
        all_keywords = []
        for paper in multilingual_papers:
            all_keywords.extend(paper['keywords'])
        
        keyword_counts = {}
        for keyword in all_keywords:
            keyword_counts[keyword] = keyword_counts.get(keyword, 0) + 1
        
        top_keywords = sorted(keyword_counts.items(), key=lambda x: x[1], reverse=True)[:10]
        print("\nTop 10 keywords found:")
        for keyword, count in top_keywords:
            print(f"  {keyword}: {count} papers")
        
    # except Exception as e:
    #     print(f"Error: {e}")
    #     return

if __name__ == "__main__":
    main()