fix bug

pages/upload.py

@@ -26,7 +26,7 @@ def process_file(file, file_type):
     st.write(f"📂 File ID: {file_id}")
 
     if file_type == "PPTX":
-        final_id = translate_pptx(file_id, file_name, source_lang='vn', target_lang='en', slides_per_batch=5)
+        final_id = translate_pptx(file_id, file_name, source_lang = source_lang, target_lang = target_lang, slides_per_batch=5)
         progress_bar.progress(60)
     elif file_type == "Excel":
         final_id = translate_xlsx(file_id = file_id, file_name = file_name, source_lang = source_lang, target_lang = target_lang)

test.ipynb

@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -235,10 +235,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Đã xóa 14 file trong collection 'root_file'\n",
-      "✅ Đã xóa 1 file trong collection 'final_pptx'\n",
-      "✅ Đã xóa 12 file trong collection 'original_xml'\n",
-      "✅ Đã xóa 8 file trong collection 'final_xml'\n"
+      "✅ Đã xóa 4 file trong collection 'root_file'\n",
+      "✅ Đã xóa 0 file trong collection 'final_pptx'\n",
+      "✅ Đã xóa 1 file trong collection 'original_xml'\n",
+      "✅ Đã xóa 1 file trong collection 'final_xml'\n"
      ]
     }
    ],
@@ -668,7 +668,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "base",
+   "display_name": "machine_translate",
    "language": "python",
    "name": "python3"
   },
@@ -682,7 +682,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.9"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

word/word_translate.ipynb

@@ -0,0 +1,580 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import docx\n",
+    "from docx import Document\n",
+    "import google.generativeai as genai\n",
+    "import ast\n",
+    "import json\n",
+    "import re\n",
+    "import time\n",
+    "\n",
+    "genai.configure(api_key=\"AIzaSyAk1LTwWMZyTfPAKmsn6JzFtI1MpnI7FH8\")\n",
+    "target_language = 'English'  \n",
+    "source_language = 'VietNamese'\n",
+    "\n",
+    "time_spent_sleeping = 0\n",
+    "mismatches = 0\n",
+    "\n",
+    "def batch_translate(texts, source_lang = 'English', target_lang=\"Vietnamese\"):\n",
+    "    \"\"\" Translates multiple text segments in a single API call. \"\"\"\n",
+    "    if not texts:\n",
+    "        return texts  # Skip if empty\n",
+    "    \n",
+    "    system_prompt = \"\"\"\n",
+    "    Translate the contents of a JSON file from the specified source language to the specified target language while preserving the structure, spaces, and context of the original text.\n",
+    "\n",
+    "        Instructions:\n",
+    "        1. You will be given three inputs: source language, target language, and a JSON file.\n",
+    "        2. The JSON file contains a Python dictionary where each key is an integer, and each value is a string.\n",
+    "        3. Ensure one-to-one correspondence—each input item must correspond to exactly one output item with the same number of items.\n",
+    "        4. Preserve spaces before or after strings. Do not remove, merge, split, or omit any strings.\n",
+    "        5. Translate paragraphs and ensure the translation makes sense when text is put together.\n",
+    "        6. Translate split words so that the word is not split in the translation.\n",
+    "        7. Return a JSON object that is a Python dictionary containing as many items as the original JSON file, with keys and order preserved.\n",
+    "        8. The output must be a syntactically correct Python dictionary.\n",
+    "\n",
+    "        Additional Examples:\n",
+    "        **Input 1**: \n",
+    "        - Source language: English\n",
+    "        - Target language: Vietnamese\n",
+    "        - JSON file: \n",
+    "        ```json\n",
+    "        {\"0\": \"My name is \", \"1\": \"Huy\", \"2\": \".\", \"3\": \" Today is \", \"4\": \"a \", \"5\": \"good day\", \"6\": \".\", \"7\": \"\"}\n",
+    "        ```\n",
+    "        **Output 1**:\n",
+    "        ```json\n",
+    "        {\"0\": \"Tên tôi là \", \"1\": \"Huy\", \"2\": \".\", \"3\": \" Hôm nay là \", \"4\": \"một \", \"5\": \"ngày đẹp\", \"6\": \".\", \"7\": \"\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Input 2**: \n",
+    "        - Source language: English\n",
+    "        - Target language: Spanish\n",
+    "        - JSON file: \n",
+    "        ```json\n",
+    "        {\"0\": \"The sky is \", \"1\": \"blue\", \"2\": \".\", \"3\": \" Water is \", \"4\": \"essential\", \"5\": \" for \", \"6\": \"life\", \"7\": \".\"}\n",
+    "        ```\n",
+    "        **Output 2**:\n",
+    "        ```json\n",
+    "        {\"0\": \"El cielo es \", \"1\": \"azul\", \"2\": \".\", \"3\": \" El agua es \", \"4\": \"esencial\", \"5\": \" para \", \"6\": \"la vida\", \"7\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Input 3**: \n",
+    "        - Source language: English\n",
+    "        - Target language: French   \n",
+    "        - JSON file: \n",
+    "        ```json\n",
+    "        {\"0\": \"The quick brown \", \"1\": \"fox \", \"2\": \"jumps \", \"3\": \"over \", \"4\": \"the \", \"5\": \"lazy \", \"6\": \"dog\", \"7\": \".\"}\n",
+    "        ```\n",
+    "        **Output 3**:\n",
+    "        ```json\n",
+    "        {\"0\": \"Le renard brun \", \"1\": \"rapide \", \"2\": \"saute \", \"3\": \"par-dessus \", \"4\": \"le \", \"5\": \"chien \", \"6\": \"paresseux\", \"7\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        Perform the translation and return the result as specified above. Do not include any additional text other than the translated JSON object.\n",
+    "    \"\"\"\n",
+    "    json_data = json.dumps({i: t for i, t in enumerate(texts)})\n",
+    "    user_prompt = f\"Source language: {source_lang}. Target language: {target_lang}. JSON file: {json_data}\" \n",
+    "  \n",
+    "    model = genai.GenerativeModel('gemini-2.0-flash')\n",
+    "    response = model.generate_content(contents = system_prompt.strip() + \"\\n\" + user_prompt.strip(), generation_config={\n",
+    "            'temperature': 1,  # Adjust temperature for desired creativity\n",
+    "            'top_p': 1,\n",
+    "            'top_k': 1,})\n",
+    "    # response_dict = ast.literal_eval(response.text.strip().strip(\"json```\").strip(\"```\").strip().strip(\"\\\"\"))\n",
+    "    # print(len(texts), len(list(response_dict.values())))\n",
+    "    # return list(response_dict.values())\n",
+    "\n",
+    "    return response\n",
+    "\n",
+    "def response_to_dict(response):\n",
+    "    return list(ast.literal_eval(response.text.strip().strip(\"json```\").strip(\"```\").strip().strip(\"\\\"\")).values())\n",
+    "\n",
+    "def fix_translate(texts, translated_text):\n",
+    "    \"\"\" Translates multiple text segments in a single API call. \"\"\"\n",
+    "    if not texts:\n",
+    "        return texts  # Skip if empty\n",
+    "    \n",
+    "    system_prompt = \"\"\"\n",
+    "    You are given the original JSON dictionary and the translated response text. Your task is to ensure that the translated text is in the correct format and has the same number of items as the original JSON dictionary.\n",
+    "\n",
+    "        Steps to follow:\n",
+    "        1. Parse the original and translated JSON dictionaries.\n",
+    "        2. Ensure that the keys in both dictionaries are strings (i.e., \"1\" instead of 1).\n",
+    "        3. Compare the number of items in both dictionaries.\n",
+    "        4. If the number of items in the translated dictionary is not equal to the number of items in the original dictionary, adjust the translated dictionary by:\n",
+    "            a. Adding missing items with empty strings if there are fewer items.\n",
+    "            b. Merging or splitting items to ensure correspondence with the original items if there are more items.\n",
+    "        5. Ensure that each item in the translated dictionary is in the correct order, with the same key as the original item.\n",
+    "        6. Preserve any leading or trailing spaces in the original strings.\n",
+    "        7. Ensure the output is a syntactically correct Python dictionary, with proper opening and closing braces.\n",
+    "        8. If the translated dictionary is already correct, return it as is.\n",
+    "        9. Return the corrected JSON dictionary in proper Python dictionary format.\n",
+    "\n",
+    "        Example Inputs and Outputs:\n",
+    "\n",
+    "        **Input:**\n",
+    "        - Original JSON dictionary:\n",
+    "        ```json\n",
+    "        {\"0\": \"My name is \", \"1\": \"Huy\", \"2\": \".\", \"3\": \" Today is \", \"4\": \"a \", \"5\": \"good day\", \"6\": \".\", \"7\": \"\"}\n",
+    "        ```\n",
+    "        - Translated response text with fewer items:\n",
+    "        ```json\n",
+    "        {\"0\": \"Tên tôi là \", \"1\": \"Huy\", \"2\": \".\", \"3\": \"Hôm nay \", \"4\": \"là một \", \"5\": \"ngày đẹp\", \"6\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Output:**\n",
+    "        ```json\n",
+    "        {\"0\": \"Tên tôi là \", \"1\": \"Huy\", \"2\": \".\", \"3\": \"Hôm nay \", \"4\": \"là một \", \"5\": \"ngày đẹp\", \"6\": \".\", \"7\": \"\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Input:**\n",
+    "        - Original JSON dictionary:\n",
+    "        ```json\n",
+    "        {\"0\": \"The sky is \", \"1\": \"blue\", \"2\": \".\", \"3\": \" Water is \", \"4\": \"essential\", \"5\": \" for \", \"6\": \"life\", \"7\": \".\"}\n",
+    "        ```\n",
+    "        - Translated response text with more items:\n",
+    "        ```json\n",
+    "        {\"0\": \"El cielo es \", \"1\": \"azul\", \"2\": \".\", \"3\": \" El agua es \", \"4\": \"esencial\", \"5\": \" para \", \"6\": \"la\", \"7\": \" vida\", \"8\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Output:**\n",
+    "        ```json\n",
+    "        {\"0\": \"El cielo es \", \"1\": \"azul\", \"2\": \".\", \"3\": \" El agua es \", \"4\": \"esencial\", \"5\": \" para \", \"6\": \"la vida\", \"7\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Input:**\n",
+    "        - Original JSON dictionary:\n",
+    "        ```json\n",
+    "        {\"0\": \"The quick brown \", \"1\": \"fox \", \"2\": \"jumps \", \"3\": \"over \", \"4\": \"the \", \"5\": \"lazy \", \"6\": \"dog\", \"7\": \".\"}\n",
+    "        ```\n",
+    "        - Translated response text with issues:\n",
+    "        ```json\n",
+    "        {\"0\": \"Le renard \", \"1\": \"brun \", 2: \"rapide \", 3: \"saute \", 4: \"par-dessus \", \"5\": \"le \", \"6\": \"chien \", \"7\": \"paresseux\", 8: \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Output:**\n",
+    "        ```json\n",
+    "        {\"0\": \"Le renard brun \", \"1\": \"rapide \", \"2\": \"saute \", \"3\": \"par-dessus \", \"4\": \"le \", \"5\": \"chien \", \"6\": \"paresseux\", \"7\": \".\"}\n",
+    "        ```\n",
+    "\n",
+    "        **Input:**\n",
+    "        - Original JSON dictionary:\n",
+    "        ```json\n",
+    "        {\"0\": \"The quick brown \", \"1\": \"fox \", \"2\": \"jumps \", \"3\": \"over \", \"4\": \"the \", \"5\": \"lazy \", \"6\": \"dog.\"}\n",
+    "        ```\n",
+    "        - Translated response text with wrong formatting:\n",
+    "        ```json\n",
+    "        {\"0\": \"Le renard brun \", \"1\": \"rapide \", \"2\": \"saute \", \"3\": \"par-dessus \", \"4\": \"le \", \"5\": \"chien \", \"6\": \"paresseux\".}\n",
+    "        ```\n",
+    "\n",
+    "        **Output:**\n",
+    "        ```json\n",
+    "        {\"0\": \"Le renard brun \", \"1\": \"rapide \", \"2\": \"saute \", \"3\": \"par-dessus \", \"4\": \"le \", \"5\": \"chien \", \"6\": \"paresseux.\"}\n",
+    "        ```\n",
+    "\n",
+    "        Perform the corrections and return the result as a properly formatted Python dictionary.\n",
+    "\"\"\"\n",
+    "    json_data = json.dumps({i: t for i, t in enumerate(texts)})\n",
+    "    user_prompt = f\"Original JSON dictionary: {json_data}. Translated response text: {translated_text}\" \n",
+    "  \n",
+    "    model = genai.GenerativeModel('gemini-2.0-flash')\n",
+    "    response = model.generate_content(contents = system_prompt.strip() + \"\\n\" + user_prompt.strip(), generation_config={\n",
+    "            'temperature': 1,  # Adjust temperature for desired creativity\n",
+    "            'top_p': 1,\n",
+    "            'top_k': 1,})\n",
+    "    return response_to_dict(response)\n",
+    "    # return response    \n",
+    "\n",
+    "def brute_force_fix(batch, translated_batch):\n",
+    "    if len(batch) > len(translated_batch):\n",
+    "        translated_batch += [\"\"] * (len(batch) - len(translated_batch))\n",
+    "    elif len(batch) < len(translated_batch):\n",
+    "        translated_batch = translated_batch[:len(batch)]\n",
+    "    return translated_batch\n",
+    "\n",
+    "def batch_translate_loop(batch, source_lang, target_lang):\n",
+    "    translated_batch_response = batch_translate(batch, source_lang, target_lang)\n",
+    "    try:\n",
+    "        translated_batch = response_to_dict(translated_batch_response)\n",
+    "        assert(len(translated_batch) == len(batch))\n",
+    "\n",
+    "    except:\n",
+    "        for i in range(10):\n",
+    "            print(f'I am ChatGPT and I am retarded, retrying translation time {i}:')\n",
+    "            try: \n",
+    "                translated_batch = fix_translate(batch, translated_batch_response.text.strip().strip(\"json```\").strip(\"```\").strip().strip(\"\\\"\"))\n",
+    "                assert(len(translated_batch) == len(batch))\n",
+    "                break\n",
+    "            except:\n",
+    "                pass\n",
+    "    try: \n",
+    "        translated_batch = fix_translate(batch, translated_batch_response.text.strip().strip(\"json```\").strip(\"```\").strip().strip(\"\\\"\"))\n",
+    "    except:\n",
+    "        try:\n",
+    "            translated_batch = response_to_dict(translated_batch_response)\n",
+    "        except:\n",
+    "            raise ValueError(\"The translated batch is not a list.\")\n",
+    "    if len(translated_batch) != len(batch):\n",
+    "        print(\"Length mismatch after translation. Brute Force Fixing...\")\n",
+    "        translated_batch = brute_force_fix(batch, translated_batch)\n",
+    "        global mismatches\n",
+    "        mismatches += 1\n",
+    "    print(len(batch), len(translated_batch))\n",
+    "    return translated_batch\n",
+    "\n",
+    "def full_translate(texts, source_lang = 'English', target_lang=\"Vietnamese\"):\n",
+    "    full_translated_texts = []\n",
+    "    batch = []\n",
+    "    word_count = 0\n",
+    "    global time_spent_sleeping\n",
+    "\n",
+    "    for string in texts:\n",
+    "        if len(string.split()) + word_count >= 2000:\n",
+    "            print('Translating a batch.')\n",
+    "\n",
+    "            translated_batch = batch_translate_loop(batch, source_lang, target_lang)\n",
+    "            full_translated_texts += translated_batch\n",
+    "            \n",
+    "            time.sleep(1)\n",
+    "            time_spent_sleeping += 1\n",
+    "            batch = []\n",
+    "            word_count = 0\n",
+    "        batch.append(string)\n",
+    "        word_count += len(string)\n",
+    "    \n",
+    "    print('Translating a batch.')\n",
+    "    if len(batch) == 0:\n",
+    "        return full_translated_texts\n",
+    "    \n",
+    "    translated_batch = batch_translate_loop(batch, source_lang, target_lang)\n",
+    "    full_translated_texts += translated_batch\n",
+    "    \n",
+    "    return full_translated_texts\n",
+    "\n",
+    "def merge_runs(runs):\n",
+    "    \"\"\" Merges adjacent runs with the same style. \"\"\"\n",
+    "    merged_runs = []\n",
+    "    for run in runs:\n",
+    "        if (merged_runs and isinstance(run, docx.text.run.Run) and isinstance(merged_runs[-1], docx.text.run.Run) and \n",
+    "            run.style == merged_runs[-1].style and \n",
+    "            merged_runs[-1].bold == run.bold and\n",
+    "            merged_runs[-1].italic == run.italic and\n",
+    "            merged_runs[-1].underline == run.underline and \n",
+    "            merged_runs[-1].font.size == run.font.size and\n",
+    "            merged_runs[-1].font.color.rgb == run.font.color.rgb and\n",
+    "            merged_runs[-1].font.name == run.font.name):\n",
+    "                merged_runs[-1].text += run.text\n",
+    "        else:\n",
+    "                merged_runs.append(run)\n",
+    "    return merged_runs\n",
+    "\n",
+    "NS_W = \"{http://schemas.openxmlformats.org/wordprocessingml/2006/main}\"\n",
+    "def translate_header_footer(doc, source_lang, target_lang):\n",
+    "    head_foot = []\n",
+    "    for section in doc.sections:\n",
+    "        for header in section.header.paragraphs:\n",
+    "            for run in header.runs:\n",
+    "                head_foot.append(run.text) \n",
+    "        for footer in section.footer.paragraphs:\n",
+    "            for run in footer.runs:\n",
+    "                head_foot.append(run.text)  \n",
+    "    translated_head_foot = full_translate(head_foot, source_lang, target_lang)\n",
+    "\n",
+    "    i = 0\n",
+    "    for section in doc.sections:\n",
+    "        for header in section.header.paragraphs:\n",
+    "            for run in header.runs:\n",
+    "                run.text = translated_head_foot[i]\n",
+    "                i += 1\n",
+    "        for footer in section.footer.paragraphs:\n",
+    "            for run in footer.runs:\n",
+    "                run.text = translated_head_foot[i]\n",
+    "                i += 1 \n",
+    "\n",
+    "def get_text_elements_para(doc):\n",
+    "    para_texts = []\n",
+    "    for para in doc.paragraphs:\n",
+    "        for element in para._element.iter():\n",
+    "            if element.tag.endswith('t'):\n",
+    "                if element.text:\n",
+    "                    emoji_pattern = r'[\\U00010000-\\U0010FFFF]'    \n",
+    "                    # Split the text but keep emojis as separate elements\n",
+    "                    parts = re.split(f'({emoji_pattern})', element.text)\n",
+    "                    for part in parts:\n",
+    "                        if re.match(emoji_pattern, part):\n",
+    "                            continue\n",
+    "                        if len(part.strip()) != 0:\n",
+    "                            para_texts.append(part)\n",
+    "\n",
+    "    return para_texts\n",
+    "\n",
+    "def get_text_elements_table(doc):\n",
+    "    table_texts = []\n",
+    "    for table in doc.tables:\n",
+    "        for row in table.rows:\n",
+    "            for cell in row.cells:\n",
+    "                table_texts += get_text_elements_para(cell)\n",
+    "    return table_texts\n",
+    "\n",
+    "def translate_paragraphs(doc, translated_texts, i = 0):\n",
+    "    for para in doc.paragraphs:\n",
+    "        for element in para._element.iter():\n",
+    "            if element.tag.endswith('t'):\n",
+    "                if element.text:\n",
+    "                    emoji_pattern = r'[\\U00010000-\\U0010FFFF]'    \n",
+    "                    # Split the text but keep emojis as separate elements\n",
+    "                    parts = re.split(f'({emoji_pattern})', element.text)\n",
+    "                    for j in range(len(parts)):\n",
+    "                        if re.match(emoji_pattern, parts[j]):\n",
+    "                            continue\n",
+    "                        if len(parts[j].strip()) != 0: \n",
+    "                            translated_text = translated_texts[i]\n",
+    "                            i += 1\n",
+    "                            parts[j] = translated_text\n",
+    "                    element.text = \"\".join(parts)                        \n",
+    "    return doc, i\n",
+    "\n",
+    "def translate_tables(doc, translated_texts):\n",
+    "    i = 0\n",
+    "    for table in doc.tables:\n",
+    "        for row in table.rows:\n",
+    "            for cell in row.cells:\n",
+    "                cell, i = translate_paragraphs(cell, translated_texts, i)\n",
+    "    return doc\n",
+    "\n",
+    "def is_same_formatting(text1, text2):\n",
+    "    \"\"\"\n",
+    "    Check if two texts have the same formatting.\n",
+    "    \"\"\"\n",
+    "    return (text1.bold == text2.bold \\\n",
+    "            and text1.italic == text2.italic \\\n",
+    "            and text1.underline == text2.underline \\\n",
+    "            and text1.font.size == text2.font.size \\\n",
+    "            and text1.font.color.rgb == text2.font.color.rgb \\\n",
+    "            and text1.font.name == text2.font.name)\n",
+    "    \n",
+    "def merge_elements(doc):\n",
+    "    for para in doc.paragraphs:\n",
+    "        current_run = []\n",
+    "        for element in para.iter_inner_content():\n",
+    "            if isinstance(element, docx.text.run.Run):\n",
+    "                if current_run  == []:\n",
+    "                    current_run = [element]\n",
+    "                elif is_same_formatting(current_run[0], element):\n",
+    "                    current_run[0].text += element.text\n",
+    "                    element.text = \"\"\n",
+    "                else:\n",
+    "                    current_run = [element]\n",
+    "    return doc\n",
+    "\n",
+    "def translate_docx(input_file, source_lang = \"English\", target_lang=\"Vietnamese\", output_num = ''):\n",
+    "    \"\"\" Translates a Word document efficiently using batch processing. \"\"\"\n",
+    "    doc = Document(input_file)\n",
+    "    output_file = os.path.join(os.path.dirname(input_file), f\"{output_num}{target_language}_translated_{os.path.basename(input_file)}\")\n",
+    "\n",
+    "    doc = merge_elements(doc)\n",
+    "    \n",
+    "    print('Translating paragraphs.')\n",
+    "    para_texts = get_text_elements_para(doc)\n",
+    "    translated_para = full_translate(para_texts, source_lang = source_lang, target_lang = target_lang)\n",
+    "    print('Done translating pararaphs.')\n",
+    "\n",
+    "    print('Translating tables.')\n",
+    "    table_texts = get_text_elements_table(doc)\n",
+    "    translated_tables = full_translate(table_texts, source_lang = source_lang, target_lang = target_lang)\n",
+    "    print('Done translating tables.')\n",
+    "\n",
+    "    print('Inserting paragaphs')\n",
+    "    doc, _ = translate_paragraphs(doc, translated_para)\n",
+    "    print('Inserting tables.')\n",
+    "    doc = translate_tables(doc, translated_tables)\n",
+    "\n",
+    "    translate_header_footer(doc, source_lang, target_lang)\n",
+    "    print('Done translating headers & footers.')\n",
+    "\n",
+    "    doc.save(output_file)\n",
+    "    print(f\"Translation complete! Saved as {output_file}\")\n",
+    "\n",
+    "    # return para_texts, translated_para\n",
+    "\n",
+    "\n",
+    "# input_file = r\"C:\\Users\\huyvu\\Downloads\\wordnet-an-electronic-lexical-database-language-speech-and-communication.9780262061971.33119-1.docx\"\n",
+    "input_file = r\"D:\\Show_me_everything\\Machine Translation\\input\\test1.docx\"\n",
+    "# input_file = r\"C:\\Users\\huyvu\\Documents\\Machine Translation\\Machine-Translation\\data\\input\\Data Engineering Practice.docx\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "input_file = r\"D:\\Show_me_everything\\Machine Translation\\input\\Tổng quan cuộc thi_ Data Unlock.docx\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Translating paragraphs.\n",
+      "Translating a batch.\n",
+      "37 37\n",
+      "Translating a batch.\n",
+      "27 27\n",
+      "Translating a batch.\n",
+      "28 28\n",
+      "Translating a batch.\n",
+      "20 20\n",
+      "Translating a batch.\n",
+      "16 16\n",
+      "Translating a batch.\n",
+      "23 23\n",
+      "Translating a batch.\n",
+      "37 37\n",
+      "Translating a batch.\n",
+      "32 32\n",
+      "Translating a batch.\n",
+      "28 28\n",
+      "Translating a batch.\n",
+      "13 13\n",
+      "Translating a batch.\n",
+      "36 36\n",
+      "Translating a batch.\n",
+      "23 23\n",
+      "Translating a batch.\n",
+      "25 25\n",
+      "Translating a batch.\n",
+      "10 10\n",
+      "Translating a batch.\n",
+      "14 14\n",
+      "Translating a batch.\n",
+      "13 13\n",
+      "Translating a batch.\n",
+      "11 11\n",
+      "Translating a batch.\n",
+      "18 18\n",
+      "Translating a batch.\n",
+      "14 14\n",
+      "Translating a batch.\n",
+      "23 23\n",
+      "Translating a batch.\n",
+      "24 24\n",
+      "Translating a batch.\n",
+      "21 21\n",
+      "Translating a batch.\n",
+      "10 10\n",
+      "Translating a batch.\n",
+      "23 23\n",
+      "Translating a batch.\n",
+      "12 12\n",
+      "Translating a batch.\n",
+      "18 18\n",
+      "Translating a batch.\n",
+      "17 17\n",
+      "Translating a batch.\n",
+      "29 29\n",
+      "Translating a batch.\n",
+      "15 15\n",
+      "Translating a batch.\n",
+      "21 21\n",
+      "Translating a batch.\n",
+      "2 2\n",
+      "Done translating pararaphs.\n",
+      "Translating tables.\n",
+      "Translating a batch.\n",
+      "21 21\n",
+      "Done translating tables.\n",
+      "Inserting paragaphs\n",
+      "Inserting tables.\n",
+      "Translating a batch.\n",
+      "Done translating headers & footers.\n",
+      "Translation complete! Saved as D:\\Show_me_everything\\Machine Translation\\input\\English_translated_test1.docx\n"
+     ]
+    }
+   ],
+   "source": [
+    "translate_docx(input_file, source_lang = source_language, target_lang = target_language)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "30"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "time_spent_sleeping"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mismatches"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "machine_translate",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

word/word_translate.py

@@ -1,3 +1,4 @@
+import os
 import docx
 from docx import Document
 import google.generativeai as genai
@@ -5,79 +6,250 @@ import ast
 import json
 import re
 import dotenv
-import os
-import io
-
 from pymongo import MongoClient
 from gridfs import GridFS
 from docx import Document
+from io import BytesIO
+
 
 dotenv.load_dotenv(".env")
 api_key = os.getenv("GEMINI_API_KEY")
-genai.configure(api_key=api_key)
-model = genai.GenerativeModel("gemini-2.0-flash")
 
-def batch_translate(texts, source_lang = "English", target_lang="Vietnamese"):
+
+def batch_translate(texts, source_lang = 'English', target_lang="Vietnamese"):
+    """ Translates multiple text segments in a single API call. """
+    if not texts:
+        return texts  # Skip if empty
+    
+    system_prompt = """
+    Translate the contents of a JSON file from the specified source language to the specified target language while preserving the structure, spaces, and context of the original text.
+
+        Instructions:
+        1. You will be given three inputs: source language, target language, and a JSON file.
+        2. The JSON file contains a Python dictionary where each key is an integer, and each value is a string.
+        3. Ensure one-to-one correspondence—each input item must correspond to exactly one output item with the same number of items.
+        4. Preserve spaces before or after strings. Do not remove, merge, split, or omit any strings.
+        5. Translate paragraphs and ensure the translation makes sense when text is put together.
+        6. Translate split words so that the word is not split in the translation.
+        7. Return a JSON object that is a Python dictionary containing as many items as the original JSON file, with keys and order preserved.
+        8. The output must be a syntactically correct Python dictionary.
+
+        Additional Examples:
+        **Input 1**: 
+        - Source language: English
+        - Target language: Vietnamese
+        - JSON file: 
+        ```json
+        {"0": "My name is ", "1": "Huy", "2": ".", "3": " Today is ", "4": "a ", "5": "good day", "6": ".", "7": ""}
+        ```
+        **Output 1**:
+        ```json
+        {"0": "Tên tôi là ", "1": "Huy", "2": ".", "3": " Hôm nay là ", "4": "một ", "5": "ngày đẹp", "6": ".", "7": ""}
+        ```
+
+        **Input 2**: 
+        - Source language: English
+        - Target language: Spanish
+        - JSON file: 
+        ```json
+        {"0": "The sky is ", "1": "blue", "2": ".", "3": " Water is ", "4": "essential", "5": " for ", "6": "life", "7": "."}
+        ```
+        **Output 2**:
+        ```json
+        {"0": "El cielo es ", "1": "azul", "2": ".", "3": " El agua es ", "4": "esencial", "5": " para ", "6": "la vida", "7": "."}
+        ```
+
+        **Input 3**: 
+        - Source language: English
+        - Target language: French   
+        - JSON file: 
+        ```json
+        {"0": "The quick brown ", "1": "fox ", "2": "jumps ", "3": "over ", "4": "the ", "5": "lazy ", "6": "dog", "7": "."}
+        ```
+        **Output 3**:
+        ```json
+        {"0": "Le renard brun ", "1": "rapide ", "2": "saute ", "3": "par-dessus ", "4": "le ", "5": "chien ", "6": "paresseux", "7": "."}
+        ```
+
+        Perform the translation and return the result as specified above. Do not include any additional text other than the translated JSON object.
+    """
+    json_data = json.dumps({i: t for i, t in enumerate(texts)})
+    user_prompt = f"Source language: {source_lang}. Target language: {target_lang}. JSON file: {json_data}" 
+  
+    model = genai.GenerativeModel('gemini-2.0-flash')
+    response = model.generate_content(contents = system_prompt.strip() + "\n" + user_prompt.strip(), generation_config={
+            'temperature': 1,  # Adjust temperature for desired creativity
+            'top_p': 1,
+            'top_k': 1,})
+    # response_dict = ast.literal_eval(response.text.strip().strip("json```").strip("```").strip().strip("\""))
+    # print(len(texts), len(list(response_dict.values())))
+    # return list(response_dict.values())
+
+    return response
+
+def response_to_dict(response):
+    return list(ast.literal_eval(response.text.strip().strip("json```").strip("```").strip().strip("\"")).values())
+
+def fix_translate(texts, translated_text):
     """ Translates multiple text segments in a single API call. """
     if not texts:
         return texts  # Skip if empty
     
-    system_prompt = """ You are given three inputs: source language, target language and a json file.
-            - Your task is to translate the JSON file from the source language (you have to detect the source language yourself) to the target language.
-            - The original JSON file contains a Python array of objects, each with "index" and "text" keys.  
-            - Ensure **one-to-one correspondence** — the output must have exactly as many items as the input.  
-            - The array contains text that makes up whole paragraphs. Make sure that the translation makes sense when the text is put together and retains the same context.
-            - This is very important: Empty spaces should be left as is. For example: From English, "Hello " should be translated into Vietnamese as "Xin chào ", with the same space at the end.
-            - Very frequently there are spaces before or after a string. Do not remove these spaces. 
-            - If the source language is English and the target language is Vietnamese and a string contains "'s" in the possessive sense, translate it as "của". 
-            - Example: [["WorldQuant's"], ["Mission"]] should be translated as [["Nhiệm vụ"], ["của WorldQuant"]]
-            - Words might be split into multiple continuous arrays. Translate them such that the translation corresponds to the full word.
-            - If a word is split up into multiple arrays, the translation should be such that the word is not split up.
-            - Exampe: ['Tesla sells its pro', 'ducts'] should be translated as ['Tesla bán sản phẩm của mình', ''.]. Note that the number of elements in the output is the same as the input.
-            - Example: [["Hello"], ["World"]] should be translated as [["Xin chào"], ["Thế giới"]]
-            - Do **not** merge, split, or omit strings. Each input object corresponds to exactly one output object.  
-            - Return a JSON object that is a Python array.
-            - Each object in the array is a dictionary with two keys: "index" and "text".
-            - The text should be the translated version of the text in the original object, and the index should stay consistent.
-            - The number of objects in the output MUST the same as the number of objects in the input.
-            - The format of the output should look exactly like the example.
-            - Example:
-            **Input**: Target language: Vietnamese. JSON file: 
-            [{"index": 0, "text": ["My name is "]}, {"index": 1, "text": ["Huy"]}, {"index": 2, "text": ["."]}, {"index": 3, "text": ["Today is "]}, {"index": 4, "text": ["a"]}, {"index": 5, "text": ["good day"]}, {"index": 6, "text": ["."]}, {"index": 7, "text": [""]}]'
-            **Output**: [{"index": 0, "text": ["Tên tôi là "]}, {"index": 1, "text": ["Huy"]}, {"index": 2, "text": ["."]}, {"index": 4, "text": ["Hôm nay là "]}, {"index": 3, "text": ["Một"]}, {"index": 5, "text": ["ngày đẹp"]}, {"index": 6, "text": ["."]}, {"index": 7, "text": [""]}]
-            - Return the result of translation according to the format. Do NOT return code for translating.
-            """  
-    json_data = json.dumps([{"index": i, "text": t} for i, t in enumerate(texts)])
-    user_prompt = f"Source languag: {source_lang}. Target language: {target_lang}. JSON file: {json_data}" 
+    system_prompt = """
+    You are given the original JSON dictionary and the translated response text. Your task is to ensure that the translated text is in the correct format and has the same number of items as the original JSON dictionary.
+
+        Steps to follow:
+        1. Parse the original and translated JSON dictionaries.
+        2. Ensure that the keys in both dictionaries are strings (i.e., "1" instead of 1).
+        3. Compare the number of items in both dictionaries.
+        4. If the number of items in the translated dictionary is not equal to the number of items in the original dictionary, adjust the translated dictionary by:
+            a. Adding missing items with empty strings if there are fewer items.
+            b. Merging or splitting items to ensure correspondence with the original items if there are more items.
+        5. Ensure that each item in the translated dictionary is in the correct order, with the same key as the original item.
+        6. Preserve any leading or trailing spaces in the original strings.
+        7. Ensure the output is a syntactically correct Python dictionary, with proper opening and closing braces.
+        8. If the translated dictionary is already correct, return it as is.
+        9. Return the corrected JSON dictionary in proper Python dictionary format.
+
+        Example Inputs and Outputs:
+
+        **Input:**
+        - Original JSON dictionary:
+        ```json
+        {"0": "My name is ", "1": "Huy", "2": ".", "3": " Today is ", "4": "a ", "5": "good day", "6": ".", "7": ""}
+        ```
+        - Translated response text with fewer items:
+        ```json
+        {"0": "Tên tôi là ", "1": "Huy", "2": ".", "3": "Hôm nay ", "4": "là một ", "5": "ngày đẹp", "6": "."}
+        ```
+
+        **Output:**
+        ```json
+        {"0": "Tên tôi là ", "1": "Huy", "2": ".", "3": "Hôm nay ", "4": "là một ", "5": "ngày đẹp", "6": ".", "7": ""}
+        ```
+
+        **Input:**
+        - Original JSON dictionary:
+        ```json
+        {"0": "The sky is ", "1": "blue", "2": ".", "3": " Water is ", "4": "essential", "5": " for ", "6": "life", "7": "."}
+        ```
+        - Translated response text with more items:
+        ```json
+        {"0": "El cielo es ", "1": "azul", "2": ".", "3": " El agua es ", "4": "esencial", "5": " para ", "6": "la", "7": " vida", "8": "."}
+        ```
+
+        **Output:**
+        ```json
+        {"0": "El cielo es ", "1": "azul", "2": ".", "3": " El agua es ", "4": "esencial", "5": " para ", "6": "la vida", "7": "."}
+        ```
+
+        **Input:**
+        - Original JSON dictionary:
+        ```json
+        {"0": "The quick brown ", "1": "fox ", "2": "jumps ", "3": "over ", "4": "the ", "5": "lazy ", "6": "dog", "7": "."}
+        ```
+        - Translated response text with issues:
+        ```json
+        {"0": "Le renard ", "1": "brun ", 2: "rapide ", 3: "saute ", 4: "par-dessus ", "5": "le ", "6": "chien ", "7": "paresseux", 8: "."}
+        ```
+
+        **Output:**
+        ```json
+        {"0": "Le renard brun ", "1": "rapide ", "2": "saute ", "3": "par-dessus ", "4": "le ", "5": "chien ", "6": "paresseux", "7": "."}
+        ```
+
+        **Input:**
+        - Original JSON dictionary:
+        ```json
+        {"0": "The quick brown ", "1": "fox ", "2": "jumps ", "3": "over ", "4": "the ", "5": "lazy ", "6": "dog."}
+        ```
+        - Translated response text with wrong formatting:
+        ```json
+        {"0": "Le renard brun ", "1": "rapide ", "2": "saute ", "3": "par-dessus ", "4": "le ", "5": "chien ", "6": "paresseux".}
+        ```
+
+        **Output:**
+        ```json
+        {"0": "Le renard brun ", "1": "rapide ", "2": "saute ", "3": "par-dessus ", "4": "le ", "5": "chien ", "6": "paresseux."}
+        ```
+
+        Perform the corrections and return the result as a properly formatted Python dictionary.
+"""
+    json_data = json.dumps({i: t for i, t in enumerate(texts)})
+    user_prompt = f"Original JSON dictionary: {json_data}. Translated response text: {translated_text}" 
   
     model = genai.GenerativeModel('gemini-2.0-flash')
     response = model.generate_content(contents = system_prompt.strip() + "\n" + user_prompt.strip(), generation_config={
             'temperature': 1,  # Adjust temperature for desired creativity
             'top_p': 1,
             'top_k': 1,})
-    response_dict = ast.literal_eval(response.text.strip().strip("json```").strip("```").strip())
-    if len(response_dict) > 0:
-        if isinstance(response_dict[0]['text'], list):
-            translated_texts = [i['text'][0] for i in sorted(response_dict, key = lambda x: x['index'])]
-        elif isinstance(response_dict[0]['text'], str):
-            translated_texts = [i['text'] for i in sorted(response_dict, key = lambda x: x['index'])]
-    return translated_texts
-
-def full_translate(texts, source_lang, target_lang="Vietnamese"):
+    return response_to_dict(response)
+    # return response    
+
+def brute_force_fix(batch, translated_batch):
+    if len(batch) > len(translated_batch):
+        translated_batch += [""] * (len(batch) - len(translated_batch))
+    elif len(batch) < len(translated_batch):
+        translated_batch = translated_batch[:len(batch)]
+    return translated_batch
+
+def batch_translate_loop(batch, source_lang, target_lang):
+    translated_batch_response = batch_translate(batch, source_lang, target_lang)
+    try:
+        translated_batch = response_to_dict(translated_batch_response)
+        assert(len(translated_batch) == len(batch))
+
+    except:
+        for i in range(10):
+            print(f'I am ChatGPT and I am retarded, retrying translation time {i}:')
+            try: 
+                translated_batch = fix_translate(batch, translated_batch_response.text.strip().strip("json```").strip("```").strip().strip("\""))
+                assert(len(translated_batch) == len(batch))
+                break
+            except:
+                pass
+    try: 
+        translated_batch = fix_translate(batch, translated_batch_response.text.strip().strip("json```").strip("```").strip().strip("\""))
+    except:
+        try:
+            translated_batch = response_to_dict(translated_batch_response)
+        except:
+            raise ValueError("The translated batch is not a list.")
+    if len(translated_batch) != len(batch):
+        print("Length mismatch after translation. Brute Force Fixing...")
+        translated_batch = brute_force_fix(batch, translated_batch)
+        global mismatches
+        mismatches += 1
+    print(len(batch), len(translated_batch))
+    return translated_batch
+
+def full_translate(texts, source_lang = 'English', target_lang="Vietnamese"):
     full_translated_texts = []
     batch = []
     word_count = 0
+    global time_spent_sleeping
 
     for string in texts:
         if len(string.split()) + word_count >= 1000:
             print('Translating a batch.')
-            full_translated_texts += batch_translate(batch, source_lang, target_lang)
+
+            translated_batch = batch_translate_loop(batch, source_lang, target_lang)
+            full_translated_texts += translated_batch
+            
+            time.sleep(3)
+            time_spent_sleeping += 3
             batch = []
             word_count = 0
         batch.append(string)
-        word_count += len(string.split())
-
-    full_translated_texts += batch_translate(batch, source_lang, target_lang)    
+        word_count += len(string)
+    
+    print('Translating a batch.')
+    if len(batch) == 0:
+        return full_translated_texts
+    
+    translated_batch = batch_translate_loop(batch, source_lang, target_lang)
+    full_translated_texts += translated_batch
+    
     return full_translated_texts
 
 def merge_runs(runs):
@@ -98,8 +270,7 @@ def merge_runs(runs):
     return merged_runs
 
 NS_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
-
-def translate_header_footer(doc, target_lang):
+def translate_header_footer(doc, source_lang, target_lang):
     head_foot = []
     for section in doc.sections:
         for header in section.header.paragraphs:
@@ -108,7 +279,7 @@ def translate_header_footer(doc, target_lang):
         for footer in section.footer.paragraphs:
             for run in footer.runs:
                 head_foot.append(run.text)  
-    translated_head_foot = full_translate(head_foot, target_lang)
+    translated_head_foot = full_translate(head_foot, source_lang, target_lang)
 
     i = 0
     for section in doc.sections:
@@ -119,8 +290,8 @@ def translate_header_footer(doc, target_lang):
         for footer in section.footer.paragraphs:
             for run in footer.runs:
                 run.text = translated_head_foot[i]
-                i += 1
-                
+                i += 1 
+
 def get_text_elements_para(doc):
     para_texts = []
     for para in doc.paragraphs:
@@ -133,7 +304,9 @@ def get_text_elements_para(doc):
                     for part in parts:
                         if re.match(emoji_pattern, part):
                             continue
-                        para_texts.append(part)
+                        if len(part.strip()) != 0:
+                            para_texts.append(part)
+
     return para_texts
 
 def get_text_elements_table(doc):
@@ -155,9 +328,10 @@ def translate_paragraphs(doc, translated_texts, i = 0):
                     for j in range(len(parts)):
                         if re.match(emoji_pattern, parts[j]):
                             continue
-                        translated_text = translated_texts[i]
-                        i += 1
-                        parts[j] = translated_text
+                        if len(parts[j].strip()) != 0: 
+                            translated_text = translated_texts[i]
+                            i += 1
+                            parts[j] = translated_text
                     element.text = "".join(parts)                        
     return doc, i
 
@@ -169,36 +343,60 @@ def translate_tables(doc, translated_texts):
                 cell, i = translate_paragraphs(cell, translated_texts, i)
     return doc
 
-def translate_docx_from_mongodb(file_id, source_lang, target_lang="Vietnamese"):
-        # Kết nối MongoDB
-    client = MongoClient("mongodb+srv://admin:1highbar456@cluster0.equkm.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0")
-    db = client["word"]
-    fs_input = GridFS(db, collection="root_file")
-    fs_output = GridFS(db, collection="final_file")
+def is_same_formatting(text1, text2):
+    """
+    Check if two texts have the same formatting.
+    """
+    return (text1.bold == text2.bold \
+            and text1.italic == text2.italic \
+            and text1.underline == text2.underline \
+            and text1.font.size == text2.font.size \
+            and text1.font.color.rgb == text2.font.color.rgb \
+            and text1.font.name == text2.font.name)
     
-    # Lấy file từ MongoDB
-    file_data = fs_input.get(file_id).read()
-    original_file = fs_input.get(file_id).filename  # Lấy tên gốc của file
-    doc = Document(io.BytesIO(file_data))
+def merge_elements(doc):
+    for para in doc.paragraphs:
+        current_run = []
+        for element in para.iter_inner_content():
+            if isinstance(element, docx.text.run.Run):
+                if current_run  == []:
+                    current_run = [element]
+                elif is_same_formatting(current_run[0], element):
+                    current_run[0].text += element.text
+                    element.text = ""
+                else:
+                    current_run = [element]
+    return doc
+
+def translate_docx(word_id, source_lang = "English", target_lang="Vietnamese", output_num = ''):
+    """ Translates a Word document efficiently using batch processing. """
+
+    client = MongoClient("mongodb+srv://admin:[email protected]/?retryWrites=true&w=majority&appName=Cluster0")
+    db = client['pptx']
+    fs = GridFS(db, collection='root_file')
+    word_file = fs.get(word_id)
+
+    doc = Document(BytesIO(word_file.read()))
+    output_file = os.path.join(os.path.dirname(input_file), f"{output_num}{target_language}_translated_{os.path.basename(input_file)}")
+
+    doc = merge_elements(doc)
     
-    # Lấy nội dung và dịch
+    print('Translating paragraphs.')
     para_texts = get_text_elements_para(doc)
-    translated_para = full_translate(para_texts, source_lang, target_lang)
-    
+    translated_para = full_translate(para_texts, source_lang = source_lang, target_lang = target_lang)
+    print('Done translating pararaphs.')
+
+    print('Translating tables.')
     table_texts = get_text_elements_table(doc)
-    translated_tables = full_translate(table_texts, source_lang, target_lang)
-    
-    # Cập nhật nội dung dịch vào document
+    translated_tables = full_translate(table_texts, source_lang = source_lang, target_lang = target_lang)
+    print('Done translating tables.')
+
+    print('Inserting paragaphs')
     doc, _ = translate_paragraphs(doc, translated_para)
+    print('Inserting tables.')
     doc = translate_tables(doc, translated_tables)
-    translate_header_footer(doc, target_lang)
-    
-    # Lưu file dịch vào MongoDB với cùng tên gốc
-    output_stream = io.BytesIO()
-    doc.save(output_stream)
-    output_stream.seek(0)
-    
-    translated_file_id = fs_output.put(output_stream, filename=original_file)
-    client.close()
-    
-    return translated_file_id
+
+    translate_header_footer(doc, source_lang, target_lang)
+    print('Done translating headers & footers.')
+
+    doc.save(output_file)