')
if current_para_count < original_para_count / 2:
log(f"⚠️ Paragraph mismatch! Original: {original_para_count}, Current: {current_para_count}")
log("🔧 Attempting emergency paragraph restoration...")
if '
' not in text and len(text) > 300:
log("❌ No paragraph tags found - applying emergency restoration")
if '\n\n' in text:
parts = text.split('\n\n')
paragraphs = ['') < 3 and len(text) > 1000:
log("⚠️ Very few paragraphs for long text - checking if more breaks needed")
soup = BeautifulSoup(text, 'html.parser')
existing_paras = soup.find_all('p')
new_paragraphs = []
for para in existing_paras:
para_text = para.get_text()
if len(para_text) > 500:
sentences = re.split(r'(?<=[.!?])\s+', para_text)
if len(sentences) > 5:
chunks = []
current = []
for sent in sentences:
current.append(sent)
if len(current) >= 3:
chunks.append('
' in content_stripped[:500] # No common HTML tags
):
# This looks like plain text or markdown from html2text
is_plain_text = True
if is_plain_text:
# For plain text, just check the length
text_length = len(content_stripped)
# Be more lenient with plain text since it's already extracted
return text_length > 50 # Much lower threshold for plain text
# Original HTML parsing logic
soup = BeautifulSoup(html_content, 'html.parser')
soup_copy = BeautifulSoup(str(soup), 'html.parser')
for img in soup_copy.find_all('img'):
img.decompose()
text_elements = soup_copy.find_all(['p', 'div', 'span'])
text_content = ' '.join(elem.get_text(strip=True) for elem in text_elements)
headers = soup_copy.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
header_text = ' '.join(h.get_text(strip=True) for h in headers)
if headers and len(text_content.strip()) > 1:
return True
if len(text_content.strip()) > 200:
return True
if len(header_text.strip()) > 100:
return True
return False
except Exception as e:
print(f"Warning: Error checking text content: {e}")
return True
# =====================================================
# UNIFIED CHAPTER EXTRACTOR
# =====================================================
class ChapterExtractor:
"""Unified chapter extraction with three modes: Smart, Comprehensive, and Full"""
def __init__(self, progress_callback=None):
self.pattern_manager = PatternManager()
self.progress_callback = progress_callback # Add progress callback
self.parser = self._get_best_parser() # Determine best parser on init
def _get_best_parser(self):
"""Determine the best parser available, preferring lxml for CJK text"""
try:
import lxml
return 'lxml'
except ImportError:
return 'html.parser'
def _sort_by_opf_spine(self, chapters, opf_path):
"""Sort chapters according to OPF spine order"""
try:
import xml.etree.ElementTree as ET
# Read OPF file
with open(opf_path, 'r', encoding='utf-8') as f:
opf_content = f.read()
# Parse OPF
root = ET.fromstring(opf_content)
# Find namespaces
ns = {'opf': 'http://www.idpf.org/2007/opf'}
if root.tag.startswith('{'):
default_ns = root.tag[1:root.tag.index('}')]
ns = {'opf': default_ns}
# Build manifest map (id -> href)
manifest = {}
for item in root.findall('.//opf:manifest/opf:item', ns):
item_id = item.get('id')
href = item.get('href')
if item_id and href:
manifest[item_id] = href
# Get spine order
spine_order = []
spine = root.find('.//opf:spine', ns)
if spine is not None:
for itemref in spine.findall('opf:itemref', ns):
idref = itemref.get('idref')
if idref and idref in manifest:
href = manifest[idref]
spine_order.append(href)
if not spine_order:
print("⚠️ No spine order found in OPF, keeping original order")
return chapters
# Create a mapping of filenames to spine position
spine_map = {}
for idx, href in enumerate(spine_order):
# Try different matching strategies
basename = os.path.basename(href)
spine_map[basename] = idx
spine_map[href] = idx
# Also store without extension for flexible matching
name_no_ext = os.path.splitext(basename)[0]
spine_map[name_no_ext] = idx
print(f"📋 OPF spine contains {len(spine_order)} items")
# Sort chapters based on spine order
def get_spine_position(chapter):
# Try to match chapter to spine
filename = chapter.get('filename', '')
basename = chapter.get('original_basename', '')
# Try exact filename match
if filename in spine_map:
return spine_map[filename]
# Try basename match
if basename in spine_map:
return spine_map[basename]
# Try basename of filename
if filename:
fname_base = os.path.basename(filename)
if fname_base in spine_map:
return spine_map[fname_base]
# Try without extension
if basename:
if basename + '.html' in spine_map:
return spine_map[basename + '.html']
if basename + '.xhtml' in spine_map:
return spine_map[basename + '.xhtml']
# Fallback to chapter number * 1000 (to sort after spine items)
return 1000000 + chapter.get('num', 0)
# Sort chapters
sorted_chapters = sorted(chapters, key=get_spine_position)
# Renumber chapters based on new order
for idx, chapter in enumerate(sorted_chapters, 1):
chapter['spine_order'] = idx
# Optionally update chapter numbers to match spine order
# chapter['num'] = idx # Uncomment if you want to renumber
# Log reordering info
reordered_count = 0
for idx, chapter in enumerate(sorted_chapters):
original_idx = chapters.index(chapter)
if original_idx != idx:
reordered_count += 1
if reordered_count > 0:
print(f"🔄 Reordered {reordered_count} chapters to match OPF spine")
else:
print(f"✅ Chapter order already matches OPF spine")
return sorted_chapters
except Exception as e:
print(f"⚠️ Could not sort by OPF spine: {e}")
import traceback
traceback.print_exc()
return chapters
def protect_angle_brackets_with_korean(self, text: str) -> str:
"""Protect CJK text in angle brackets from HTML parsing"""
if text is None:
return ""
import re
# Extended pattern to include Korean, Chinese, and Japanese characters
cjk_pattern = r'[가-힣ㄱ-ㅎㅏ-ㅣ一-龿ぁ-ゟァ-ヿ]'
bracket_pattern = rf'<([^<>]*{cjk_pattern}[^<>]*)>'
def replace_brackets(match):
content = match.group(1)
return f'<{content}>'
return re.sub(bracket_pattern, replace_brackets, text)
def ensure_all_opf_chapters_extracted(zf, chapters, out):
"""Ensure ALL chapters from OPF spine are extracted, not just what ChapterExtractor found"""
# Parse OPF to get ALL chapters in spine
opf_chapters = []
try:
# Find content.opf
opf_content = None
for name in zf.namelist():
if name.endswith('content.opf'):
opf_content = zf.read(name)
break
if not opf_content:
return chapters # No OPF, return original
import xml.etree.ElementTree as ET
root = ET.fromstring(opf_content)
# Handle namespaces
ns = {'opf': 'http://www.idpf.org/2007/opf'}
if root.tag.startswith('{'):
default_ns = root.tag[1:root.tag.index('}')]
ns = {'opf': default_ns}
# Get manifest
manifest = {}
for item in root.findall('.//opf:manifest/opf:item', ns):
item_id = item.get('id')
href = item.get('href')
media_type = item.get('media-type', '')
if item_id and href and ('html' in media_type.lower() or href.endswith(('.html', '.xhtml', '.htm'))):
manifest[item_id] = href
# Get spine order
spine = root.find('.//opf:spine', ns)
if spine:
for itemref in spine.findall('opf:itemref', ns):
idref = itemref.get('idref')
if idref and idref in manifest:
href = manifest[idref]
filename = os.path.basename(href)
# Skip nav, toc, cover
if any(skip in filename.lower() for skip in ['nav', 'toc', 'cover']):
continue
opf_chapters.append(href)
print(f"📚 OPF spine contains {len(opf_chapters)} chapters")
# Check which OPF chapters are missing from extraction
extracted_files = set()
for c in chapters:
if 'filename' in c:
extracted_files.add(c['filename'])
if 'original_basename' in c:
extracted_files.add(c['original_basename'])
missing_chapters = []
for opf_chapter in opf_chapters:
basename = os.path.basename(opf_chapter)
if basename not in extracted_files and opf_chapter not in extracted_files:
missing_chapters.append(opf_chapter)
if missing_chapters:
print(f"⚠️ {len(missing_chapters)} chapters in OPF but not extracted!")
print(f" Missing: {missing_chapters[:5]}{'...' if len(missing_chapters) > 5 else ''}")
# Extract the missing chapters
for href in missing_chapters:
try:
# Read the chapter content
content = zf.read(href).decode('utf-8')
# Extract chapter number
import re
basename = os.path.basename(href)
matches = re.findall(r'(\d+)', basename)
if matches:
chapter_num = int(matches[-1])
else:
chapter_num = len(chapters) + 1
# Create chapter entry
from bs4 import BeautifulSoup
parser = 'lxml' if 'lxml' in sys.modules else 'html.parser'
soup = BeautifulSoup(content, parser)
# Get title
title = "Chapter " + str(chapter_num)
title_tag = soup.find('title')
if title_tag:
title = title_tag.get_text().strip() or title
else:
for tag in ['h1', 'h2', 'h3']:
header = soup.find(tag)
if header:
title = header.get_text().strip() or title
break
# Save the chapter file
output_filename = f"chapter_{chapter_num:04d}_{basename}"
output_path = os.path.join(out, output_filename)
with open(output_path, 'w', encoding='utf-8') as f:
f.write(content)
# Add to chapters list
new_chapter = {
'num': chapter_num,
'title': title,
'body': content,
'filename': href,
'original_basename': basename,
'file_size': len(content),
'has_images': bool(soup.find_all('img')),
'detection_method': 'opf_recovery',
'content_hash': None # Will be calculated later
}
chapters.append(new_chapter)
print(f" ✅ Recovered chapter {chapter_num}: {basename}")
except Exception as e:
print(f" ❌ Failed to extract {href}: {e}")
# Re-sort chapters by number
chapters.sort(key=lambda x: x['num'])
print(f"✅ Total chapters after OPF recovery: {len(chapters)}")
except Exception as e:
print(f"⚠️ Error checking OPF chapters: {e}")
import traceback
traceback.print_exc()
return chapters
def extract_chapters(self, zf, output_dir):
"""Extract chapters and all resources from EPUB using ThreadPoolExecutor"""
import time
# Check stop at the very beginning
if is_stop_requested():
print("❌ Extraction stopped by user")
return []
print("🚀 Starting EPUB extraction with ThreadPoolExecutor...")
print(f"📄 Using parser: {self.parser} {'(optimized for CJK)' if self.parser == 'lxml' else '(standard)'}")
# Initial progress
if self.progress_callback:
self.progress_callback("Starting EPUB extraction...")
# First, extract and save content.opf for reference
for name in zf.namelist():
if name.endswith('.opf'):
try:
opf_content = zf.read(name).decode('utf-8', errors='ignore')
opf_output_path = os.path.join(output_dir, 'content.opf')
with open(opf_output_path, 'w', encoding='utf-8') as f:
f.write(opf_content)
print(f"📋 Saved OPF file: {name} → content.opf")
break
except Exception as e:
print(f"⚠️ Could not save OPF file: {e}")
# Get extraction mode from environment
extraction_mode = os.getenv("EXTRACTION_MODE", "smart").lower()
print(f"✅ Using {extraction_mode.capitalize()} extraction mode")
# Get number of workers from environment or use default
max_workers = int(os.getenv("EXTRACTION_WORKERS", "2"))
print(f"🔧 Using {max_workers} workers for parallel processing")
extracted_resources = self._extract_all_resources(zf, output_dir)
# Check stop after resource extraction
if is_stop_requested():
print("❌ Extraction stopped by user")
return []
metadata_path = os.path.join(output_dir, 'metadata.json')
if os.path.exists(metadata_path):
print("📋 Loading existing metadata...")
with open(metadata_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
else:
print("📋 Extracting fresh metadata...")
metadata = self._extract_epub_metadata(zf)
print(f"📋 Extracted metadata: {list(metadata.keys())}")
chapters, detected_language = self._extract_chapters_universal(zf, extraction_mode)
# Sort chapters according to OPF spine order if available
opf_path = os.path.join(output_dir, 'content.opf')
if os.path.exists(opf_path) and chapters:
print("📋 Sorting chapters according to OPF spine order...")
chapters = self._sort_by_opf_spine(chapters, opf_path)
print(f"✅ Chapters sorted according to OPF reading order")
# Check stop after chapter extraction
if is_stop_requested():
print("❌ Extraction stopped by user")
return []
if not chapters:
print("❌ No chapters could be extracted!")
return []
chapters_info_path = os.path.join(output_dir, 'chapters_info.json')
chapters_info = []
chapters_info_lock = threading.Lock()
def process_chapter(chapter):
"""Process a single chapter"""
# Check stop in worker
if is_stop_requested():
return None
info = {
'num': chapter['num'],
'title': chapter['title'],
'original_filename': chapter.get('filename', ''),
'has_images': chapter.get('has_images', False),
'image_count': chapter.get('image_count', 0),
'text_length': chapter.get('file_size', len(chapter.get('body', ''))),
'detection_method': chapter.get('detection_method', 'unknown'),
'content_hash': chapter.get('content_hash', '')
}
if chapter.get('has_images'):
try:
soup = BeautifulSoup(chapter.get('body', ''), self.parser)
images = soup.find_all('img')
info['images'] = [img.get('src', '') for img in images]
except:
info['images'] = []
return info
# Process chapters in parallel
print(f"🔄 Processing {len(chapters)} chapters in parallel...")
if self.progress_callback:
self.progress_callback(f"Processing {len(chapters)} chapters...")
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all tasks
future_to_chapter = {
executor.submit(process_chapter, chapter): chapter
for chapter in chapters
}
# Process completed tasks
completed = 0
for future in as_completed(future_to_chapter):
if is_stop_requested():
print("❌ Extraction stopped by user")
# Cancel remaining futures
for f in future_to_chapter:
f.cancel()
return []
try:
result = future.result()
if result:
with chapters_info_lock:
chapters_info.append(result)
completed += 1
# Yield to GUI periodically (can be disabled for max speed)
if completed % 5 == 0 and os.getenv("ENABLE_GUI_YIELD", "1") == "1":
time.sleep(0.001)
# Progress updates
if completed % 10 == 0 or completed == len(chapters):
progress_msg = f"Processed {completed}/{len(chapters)} chapters"
print(f" 📊 {progress_msg}")
if self.progress_callback:
self.progress_callback(progress_msg)
except Exception as e:
chapter = future_to_chapter[future]
print(f" ❌ Error processing chapter {chapter['num']}: {e}")
# Sort chapters_info by chapter number to maintain order
chapters_info.sort(key=lambda x: x['num'])
print(f"✅ Successfully processed {len(chapters_info)} chapters")
with open(chapters_info_path, 'w', encoding='utf-8') as f:
json.dump(chapters_info, f, ensure_ascii=False, indent=2)
print(f"💾 Saved detailed chapter info to: chapters_info.json")
metadata.update({
'chapter_count': len(chapters),
'detected_language': detected_language,
'extracted_resources': extracted_resources,
'extraction_mode': extraction_mode,
'extraction_summary': {
'total_chapters': len(chapters),
'chapter_range': f"{chapters[0]['num']}-{chapters[-1]['num']}",
'resources_extracted': sum(len(files) for files in extracted_resources.values())
}
})
metadata['chapter_titles'] = {
str(c['num']): c['title'] for c in chapters
}
with open(metadata_path, 'w', encoding='utf-8') as f:
json.dump(metadata, f, ensure_ascii=False, indent=2)
print(f"💾 Saved comprehensive metadata to: {metadata_path}")
self._create_extraction_report(output_dir, metadata, chapters, extracted_resources)
self._log_extraction_summary(chapters, extracted_resources, detected_language)
print(f"🔍 VERIFICATION: {extraction_mode.capitalize()} chapter extraction completed successfully")
print(f"⚡ Used {max_workers} workers for parallel processing")
return chapters
def _extract_all_resources(self, zf, output_dir):
"""Extract all resources with parallel processing"""
import time
extracted_resources = {
'css': [],
'fonts': [],
'images': [],
'epub_structure': [],
'other': []
}
# Check if already extracted
extraction_marker = os.path.join(output_dir, '.resources_extracted')
if os.path.exists(extraction_marker):
print("📦 Resources already extracted, skipping...")
return self._count_existing_resources(output_dir, extracted_resources)
self._cleanup_old_resources(output_dir)
# Create directories
for resource_type in ['css', 'fonts', 'images']:
os.makedirs(os.path.join(output_dir, resource_type), exist_ok=True)
print(f"📦 Extracting resources in parallel...")
# Get list of files to process
file_list = [f for f in zf.namelist() if not f.endswith('/') and os.path.basename(f)]
# Thread-safe lock for extracted_resources
resource_lock = threading.Lock()
def extract_single_resource(file_path):
if is_stop_requested():
return None
try:
file_data = zf.read(file_path)
resource_info = self._categorize_resource(file_path, os.path.basename(file_path))
if resource_info:
resource_type, target_dir, safe_filename = resource_info
target_path = os.path.join(output_dir, target_dir, safe_filename) if target_dir else os.path.join(output_dir, safe_filename)
with open(target_path, 'wb') as f:
f.write(file_data)
# Thread-safe update
with resource_lock:
extracted_resources[resource_type].append(safe_filename)
return (resource_type, safe_filename)
except Exception as e:
print(f"[WARNING] Failed to extract {file_path}: {e}")
return None
# Process files in parallel
total_resources = len(file_list)
extracted_count = 0
# Use same worker count as chapter processing
resource_workers = int(os.getenv("EXTRACTION_WORKERS", "2"))
with ThreadPoolExecutor(max_workers=resource_workers) as executor:
futures = {executor.submit(extract_single_resource, file_path): file_path
for file_path in file_list}
for future in as_completed(futures):
if is_stop_requested():
executor.shutdown(wait=False)
break
extracted_count += 1
# Progress update every 20 files
if extracted_count % 20 == 0 and self.progress_callback:
self.progress_callback(f"Extracting resources: {extracted_count}/{total_resources}")
# Yield to GUI periodically (can be disabled for max speed)
if extracted_count % 10 == 0 and os.getenv("ENABLE_GUI_YIELD", "1") == "1":
time.sleep(0.001)
result = future.result()
if result:
resource_type, filename = result
# Only print for important resources
if extracted_count < 10 or resource_type in ['css', 'fonts']:
print(f" 📄 Extracted {resource_type}: {filename}")
# Mark as complete
with open(extraction_marker, 'w') as f:
f.write(f"Resources extracted at {time.time()}")
self._validate_critical_files(output_dir, extracted_resources)
return extracted_resources
def _extract_chapters_universal(self, zf, extraction_mode="smart"):
"""Universal chapter extraction with four modes: smart, comprehensive, full, enhanced
All modes now properly merge Section/Chapter pairs
Enhanced mode uses html2text for superior text processing
Now with parallel processing for improved performance
"""
# Check stop at the beginning
if is_stop_requested():
print("❌ Chapter extraction stopped by user")
return [], 'unknown'
# Import time for yielding
import time
# Initialize enhanced extractor if using enhanced mode
enhanced_extractor = None
enhanced_filtering = extraction_mode # Default fallback
preserve_structure = True
# Independent control: translate cover.html when requested
translate_cover_html = os.getenv("TRANSLATE_COVER_HTML", "0") == "1"
if extraction_mode == "enhanced":
print("🚀 Initializing Enhanced extraction mode with html2text...")
# Get enhanced mode configuration from environment
enhanced_filtering = os.getenv("ENHANCED_FILTERING", "smart")
# Avoid 'full' with html2text to prevent XML declaration artifacts; use 'comprehensive' instead
if str(enhanced_filtering).lower() == 'full':
enhanced_filtering = 'comprehensive'
preserve_structure = os.getenv("ENHANCED_PRESERVE_STRUCTURE", "1") == "1"
print(f" • Enhanced filtering level: {enhanced_filtering}")
print(f" • Preserve structure: {preserve_structure}")
# Try to initialize enhanced extractor
try:
# Import our enhanced extractor (assume it's in the same directory or importable)
from enhanced_text_extractor import EnhancedTextExtractor
enhanced_extractor = EnhancedTextExtractor(
filtering_mode=enhanced_filtering,
preserve_structure=preserve_structure
)
print("✅ Enhanced text extractor initialized successfully")
except ImportError as e:
print(f"❌ Enhanced text extractor module not found: {e}")
print(f"❌ Cannot use enhanced extraction mode. Please install enhanced_text_extractor or select a different extraction mode.")
raise e
except Exception as e:
print(f"❌ Enhanced extractor initialization failed: {e}")
print(f"❌ Cannot use enhanced extraction mode. Please select a different extraction mode.")
raise e
chapters = []
sample_texts = []
# First phase: Collect HTML files
html_files = []
file_list = zf.namelist()
total_files = len(file_list)
# Update progress for file collection
if self.progress_callback and total_files > 100:
self.progress_callback(f"Scanning {total_files} files in EPUB...")
for idx, name in enumerate(file_list):
# Check stop while collecting files
if is_stop_requested():
print("❌ Chapter extraction stopped by user")
return [], 'unknown'
# Yield to GUI every 50 files (can be disabled for max speed)
if idx % 50 == 0 and idx > 0:
if os.getenv("ENABLE_GUI_YIELD", "1") == "1":
time.sleep(0.001) # Brief yield to GUI
if self.progress_callback and total_files > 100:
self.progress_callback(f"Scanning files: {idx}/{total_files}")
if name.lower().endswith(('.xhtml', '.html', '.htm')):
# Skip cover files by default unless override is enabled
basename = os.path.basename(name).lower()
if basename in ['cover.html', 'cover.xhtml', 'cover.htm'] and not translate_cover_html:
print(f"[SKIP] Cover file excluded from all modes: {name}")
continue
# Apply filtering based on the actual extraction mode (or enhanced_filtering for enhanced mode)
current_filtering = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
if current_filtering == "smart":
# Smart mode: aggressive filtering
lower_name = name.lower()
if any(skip in lower_name for skip in [
'nav', 'toc', 'contents', 'title', 'index',
'copyright', 'acknowledgment', 'dedication'
]):
continue
elif current_filtering == "comprehensive":
# Comprehensive mode: moderate filtering
skip_keywords = ['nav.', 'toc.', 'contents.', 'copyright.']
basename = os.path.basename(name.lower())
should_skip = False
for skip in skip_keywords:
if basename == skip + 'xhtml' or basename == skip + 'html' or basename == skip + 'htm':
should_skip = True
break
if should_skip:
print(f"[SKIP] Navigation/TOC file: {name}")
continue
# else: full mode - no filtering at all (except cover which is filtered above)
html_files.append(name)
# Update mode description to include enhanced mode
mode_description = {
"smart": "potential content files",
"comprehensive": "HTML files",
"full": "ALL HTML/XHTML files (no filtering)",
"enhanced": f"files (enhanced with {enhanced_filtering} filtering)"
}
print(f"📚 Found {len(html_files)} {mode_description.get(extraction_mode, 'files')} in EPUB")
# Sort files to ensure proper order
html_files.sort()
# Check if merging is disabled via environment variable
disable_merging = os.getenv("DISABLE_CHAPTER_MERGING", "0") == "1"
processed_files = set()
merge_candidates = {} # Store potential merges without reading files yet
if disable_merging:
print("📌 Chapter merging is DISABLED - processing all files independently")
else:
print("📌 Chapter merging is ENABLED")
# Only do merging logic if not disabled
file_groups = {}
# Group files by their base number to detect Section/Chapter pairs
for file_path in html_files:
filename = os.path.basename(file_path)
# Try different patterns to extract base number
base_num = None
# Pattern 1: "No00014" from "No00014Section.xhtml"
match = re.match(r'(No\d+)', filename)
if match:
base_num = match.group(1)
else:
# Pattern 2: "0014" from "0014_section.html" or "0014_chapter.html"
match = re.match(r'^(\d+)[_\-]', filename)
if match:
base_num = match.group(1)
else:
# Pattern 3: Just numbers at the start
match = re.match(r'^(\d+)', filename)
if match:
base_num = match.group(1)
if base_num:
if base_num not in file_groups:
file_groups[base_num] = []
file_groups[base_num].append(file_path)
# Identify merge candidates WITHOUT reading files yet
for base_num, group_files in sorted(file_groups.items()):
if len(group_files) == 2:
# Check if we have a Section/Chapter pair based on filenames only
section_file = None
chapter_file = None
for file_path in group_files:
basename = os.path.basename(file_path)
# More strict detection - must have 'section' or 'chapter' in the filename
if 'section' in basename.lower() and 'chapter' not in basename.lower():
section_file = file_path
elif 'chapter' in basename.lower() and 'section' not in basename.lower():
chapter_file = file_path
if section_file and chapter_file:
# Store as potential merge candidate
merge_candidates[chapter_file] = section_file
processed_files.add(section_file)
print(f"[DEBUG] Potential merge candidate: {base_num}")
print(f" Section: {os.path.basename(section_file)}")
print(f" Chapter: {os.path.basename(chapter_file)}")
# Filter out section files that were marked for merging
files_to_process = []
for file_path in html_files:
if not disable_merging and file_path in processed_files:
print(f"[DEBUG] Skipping section file: {file_path}")
continue
files_to_process.append(file_path)
print(f"📚 Processing {len(files_to_process)} files after merge analysis")
# Thread-safe collections
sample_texts_lock = threading.Lock()
file_size_groups_lock = threading.Lock()
h1_count_lock = threading.Lock()
h2_count_lock = threading.Lock()
# Initialize counters
file_size_groups = {}
h1_count = 0
h2_count = 0
processed_count = 0
processed_count_lock = threading.Lock()
# Progress tracking
total_files = len(files_to_process)
# Function to process a single HTML file
def process_single_html_file(file_path, file_index):
nonlocal h1_count, h2_count, processed_count
# Check stop
if is_stop_requested():
return None
# Update progress
with processed_count_lock:
processed_count += 1
current_count = processed_count
if self.progress_callback and current_count % 5 == 0:
progress_msg = f"Processing chapters: {current_count}/{total_files} ({current_count*100//total_files}%)"
self.progress_callback(progress_msg)
try:
# Read file data
file_data = zf.read(file_path)
# Decode the file data
html_content = None
detected_encoding = None
for encoding in ['utf-8', 'utf-16', 'gb18030', 'shift_jis', 'euc-kr', 'gbk', 'big5']:
try:
html_content = file_data.decode(encoding)
detected_encoding = encoding
break
except UnicodeDecodeError:
continue
if not html_content:
print(f"[WARNING] Could not decode {file_path}")
return None
# Check if this file needs merging
if not disable_merging and file_path in merge_candidates:
section_file = merge_candidates[file_path]
print(f"[DEBUG] Processing merge for: {file_path}")
try:
# Read section file
section_data = zf.read(section_file)
section_html = None
for encoding in ['utf-8', 'utf-16', 'gb18030', 'shift_jis', 'euc-kr', 'gbk', 'big5']:
try:
section_html = section_data.decode(encoding)
break
except UnicodeDecodeError:
continue
if section_html:
# Quick check if section is small enough to merge
section_soup = BeautifulSoup(section_html, self.parser)
section_text = section_soup.get_text(strip=True)
if len(section_text) < 200: # Merge if section is small
# Extract body content
chapter_soup = BeautifulSoup(html_content, self.parser)
if section_soup.body:
section_body_content = ''.join(str(child) for child in section_soup.body.children)
else:
section_body_content = section_html
if chapter_soup.body:
chapter_body_content = ''.join(str(child) for child in chapter_soup.body.children)
else:
chapter_body_content = html_content
# Merge content
html_content = section_body_content + "\n
\n" + chapter_body_content
print(f" → MERGED: Section ({len(section_text)} chars) + Chapter")
else:
print(f" → NOT MERGED: Section too large ({len(section_text)} chars)")
# Remove from processed files so it gets processed separately
processed_files.discard(section_file)
except Exception as e:
print(f"[WARNING] Failed to merge {file_path}: {e}")
# === ENHANCED EXTRACTION POINT ===
# Initialize variables that will be set by extraction
content_html = None
content_text = None
chapter_title = None
enhanced_extraction_used = False
# Determine whether to use enhanced extractor based on toggle and provider
use_enhanced = enhanced_extractor and extraction_mode == "enhanced"
force_bs_traditional = False
try:
force_bs = os.getenv('FORCE_BS_FOR_TRADITIONAL', '0') == '1'
model_env = os.getenv('MODEL', '')
if force_bs and is_traditional_translation_api(model_env):
use_enhanced = False
force_bs_traditional = True
except Exception:
pass
# Use enhanced extractor if available and allowed
if use_enhanced:
print(f"🚀 Using enhanced extraction for: {os.path.basename(file_path)}")
# Get clean text from html2text
clean_content, _, chapter_title = enhanced_extractor.extract_chapter_content(
html_content, enhanced_filtering
)
enhanced_extraction_used = True
print(f"✅ Enhanced extraction complete: {len(clean_content)} chars")
# For enhanced mode, store the markdown/plain text
# This will be sent to the translation API as-is
content_html = clean_content # This is MARKDOWN/PLAIN TEXT from html2text
content_text = clean_content # Same clean text for analysis
# BeautifulSoup method (only for non-enhanced modes)
if not enhanced_extraction_used:
if extraction_mode == "enhanced" and not force_bs_traditional:
# Enhanced mode failed - skip this file
print(f"❌ Skipping {file_path} - enhanced extraction required but not available")
return None
# Parse the (possibly merged) content
protected_html = self.protect_angle_brackets_with_korean(html_content)
# Use lxml parser which handles both HTML and XHTML well
soup = BeautifulSoup(protected_html, self.parser)
# Get effective mode for filtering
effective_filtering = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
# In full mode, keep the entire HTML structure
if effective_filtering == "full":
content_html = html_content # Keep EVERYTHING
content_text = soup.get_text(strip=True)
else:
# Smart and comprehensive modes extract body content
if soup.body:
content_html = str(soup.body)
content_text = soup.body.get_text(strip=True)
else:
content_html = html_content
content_text = soup.get_text(strip=True)
# Extract title (with ignore settings support)
chapter_title = None
# Check ignore settings for batch translation
batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
# Extract from title tag if not ignored
if not ignore_title_tag and soup.title and soup.title.string:
chapter_title = soup.title.string.strip()
# Extract from header tags if not ignored and no title found
if not chapter_title and not ignore_header_tags:
for header_tag in ['h1', 'h2', 'h3']:
header = soup.find(header_tag)
if header:
chapter_title = header.get_text(strip=True)
break
# Fallback to filename if nothing found
if not chapter_title:
chapter_title = os.path.splitext(os.path.basename(file_path))[0]
# Get the effective extraction mode for processing logic
effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
# Skip truly empty files in smart mode
# BUT: Never skip anything when merging is disabled (to ensure section files are processed)
if effective_mode == "smart" and not disable_merging and len(content_text.strip()) < 10:
print(f"[SKIP] Nearly empty file: {file_path} ({len(content_text)} chars)")
return None
# Get actual chapter number based on original position
actual_chapter_num = files_to_process.index(file_path) + 1
# Mode-specific logic
if effective_mode == "comprehensive" or effective_mode == "full":
# For comprehensive/full mode, use sequential numbering
chapter_num = actual_chapter_num
if not chapter_title:
chapter_title = os.path.splitext(os.path.basename(file_path))[0]
detection_method = f"{extraction_mode}_sequential" if extraction_mode == "enhanced" else f"{effective_mode}_sequential"
elif effective_mode == "smart":
# For smart mode, when merging is disabled, use sequential numbering
if disable_merging:
chapter_num = actual_chapter_num
if not chapter_title:
chapter_title = os.path.splitext(os.path.basename(file_path))[0]
detection_method = f"{extraction_mode}_sequential_no_merge" if extraction_mode == "enhanced" else "sequential_no_merge"
else:
# When merging is enabled, try to extract chapter info
protected_html = self.protect_angle_brackets_with_korean(html_content)
soup = BeautifulSoup(protected_html, self.parser)
# Count headers (thread-safe)
h1_tags = soup.find_all('h1')
h2_tags = soup.find_all('h2')
if h1_tags:
with h1_count_lock:
h1_count += 1
if h2_tags:
with h2_count_lock:
h2_count += 1
# Try to extract chapter number and title
chapter_num, extracted_title, detection_method = self._extract_chapter_info(
soup, file_path, content_text, html_content
)
# Use extracted title if we don't have one
if extracted_title and not chapter_title:
chapter_title = extracted_title
# For hash-based filenames, chapter_num might be None
if chapter_num is None:
chapter_num = actual_chapter_num # Use actual chapter count
detection_method = f"{extraction_mode}_sequential_fallback" if extraction_mode == "enhanced" else "sequential_fallback"
print(f"[DEBUG] No chapter number found in {file_path}, assigning: {chapter_num}")
# Filter content_html for ignore settings (before processing)
batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
if (ignore_title_tag or ignore_header_tags) and content_html and not enhanced_extraction_used:
# Parse the content HTML to remove ignored tags
content_soup = BeautifulSoup(content_html, self.parser)
# Remove title tags if ignored
if ignore_title_tag:
for title_tag in content_soup.find_all('title'):
title_tag.decompose()
# Remove header tags if ignored
if ignore_header_tags:
for header_tag in content_soup.find_all(['h1', 'h2', 'h3']):
header_tag.decompose()
# Update content_html with filtered version
content_html = str(content_soup)
# Process images and metadata (same for all modes)
protected_html = self.protect_angle_brackets_with_korean(html_content)
soup = BeautifulSoup(protected_html, self.parser)
images = soup.find_all('img')
has_images = len(images) > 0
is_image_only_chapter = has_images and len(content_text.strip()) < 500
if is_image_only_chapter:
print(f"[DEBUG] Image-only chapter detected: {file_path} ({len(images)} images, {len(content_text)} chars)")
content_hash = ContentProcessor.get_content_hash(content_html)
# Collect file size groups for smart mode (thread-safe)
if effective_mode == "smart":
file_size = len(content_text)
with file_size_groups_lock:
if file_size not in file_size_groups:
file_size_groups[file_size] = []
file_size_groups[file_size].append(file_path)
# Collect sample texts (thread-safe)
with sample_texts_lock:
if len(sample_texts) < 5:
sample_texts.append(content_text[:1000])
# Ensure chapter_num is always an integer
if isinstance(chapter_num, float):
chapter_num = int(chapter_num)
# Create chapter info
chapter_info = {
"num": chapter_num, # Now guaranteed to have a value
"title": chapter_title or f"Chapter {chapter_num}",
"body": content_html,
"filename": file_path,
"original_filename": os.path.basename(file_path),
"original_basename": os.path.splitext(os.path.basename(file_path))[0],
"content_hash": content_hash,
"detection_method": detection_method if detection_method else "pending",
"file_size": len(content_text),
"has_images": has_images,
"image_count": len(images),
"is_empty": len(content_text.strip()) == 0,
"is_image_only": is_image_only_chapter,
"extraction_mode": extraction_mode,
"file_index": file_index # Store original file index for sorting
}
# Add enhanced extraction info if used
if enhanced_extraction_used:
chapter_info["enhanced_extraction"] = True
chapter_info["enhanced_filtering"] = enhanced_filtering
chapter_info["preserve_structure"] = preserve_structure
# Add merge info if applicable
if not disable_merging and file_path in merge_candidates:
chapter_info["was_merged"] = True
chapter_info["merged_with"] = merge_candidates[file_path]
if effective_mode == "smart":
chapter_info["language_sample"] = content_text[:500]
# Debug for section files
if 'section' in chapter_info['original_basename'].lower():
print(f"[DEBUG] Added section file to candidates: {chapter_info['original_basename']} (size: {chapter_info['file_size']})")
return chapter_info
except Exception as e:
print(f"[ERROR] Failed to process {file_path}: {e}")
import traceback
traceback.print_exc()
return None
# Process files in parallel or sequentially based on file count
print(f"🚀 Processing {len(files_to_process)} HTML files...")
# Initial progress
if self.progress_callback:
self.progress_callback(f"Processing {len(files_to_process)} chapters...")
candidate_chapters = [] # For smart mode
chapters_direct = [] # For other modes
# Decide whether to use parallel processing
use_parallel = len(files_to_process) > 10
if use_parallel:
# Get worker count from environment variable
max_workers = int(os.getenv("EXTRACTION_WORKERS", "2"))
print(f"📦 Using parallel processing with {max_workers} workers...")
# Process files in parallel
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all files for processing
future_to_file = {
executor.submit(process_single_html_file, file_path, idx): (file_path, idx)
for idx, file_path in enumerate(files_to_process)
}
# Collect results as they complete
for future in as_completed(future_to_file):
if is_stop_requested():
print("❌ Chapter processing stopped by user")
executor.shutdown(wait=False)
return [], 'unknown'
try:
chapter_info = future.result()
if chapter_info:
effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
# For smart mode when merging is enabled, collect candidates
# Otherwise, add directly to chapters
if effective_mode == "smart" and not disable_merging:
candidate_chapters.append(chapter_info)
else:
chapters_direct.append(chapter_info)
except Exception as e:
file_path, idx = future_to_file[future]
print(f"[ERROR] Thread error processing {file_path}: {e}")
else:
print("📦 Using sequential processing (small file count)...")
# Process files sequentially for small EPUBs
for idx, file_path in enumerate(files_to_process):
if is_stop_requested():
print("❌ Chapter processing stopped by user")
return [], 'unknown'
chapter_info = process_single_html_file(file_path, idx)
if chapter_info:
effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
# For smart mode when merging is enabled, collect candidates
# Otherwise, add directly to chapters
if effective_mode == "smart" and not disable_merging:
candidate_chapters.append(chapter_info)
else:
chapters_direct.append(chapter_info)
# Final progress update
if self.progress_callback:
self.progress_callback(f"Chapter processing complete: {len(candidate_chapters) + len(chapters_direct)} chapters")
# Sort direct chapters by file index to maintain order
chapters_direct.sort(key=lambda x: x["file_index"])
# Post-process smart mode candidates (only when merging is enabled)
effective_mode = enhanced_filtering if extraction_mode == "enhanced" else extraction_mode
if effective_mode == "smart" and candidate_chapters and not disable_merging:
# Check stop before post-processing
if is_stop_requested():
print("❌ Chapter post-processing stopped by user")
return chapters, 'unknown'
print(f"\n[SMART MODE] Processing {len(candidate_chapters)} candidate files...")
# Sort candidates by file index to maintain order
candidate_chapters.sort(key=lambda x: x["file_index"])
# Debug: Show what files we have
section_files = [c for c in candidate_chapters if 'section' in c['original_basename'].lower()]
chapter_files = [c for c in candidate_chapters if 'chapter' in c['original_basename'].lower() and 'section' not in c['original_basename'].lower()]
other_files = [c for c in candidate_chapters if c not in section_files and c not in chapter_files]
print(f" 📊 File breakdown:")
print(f" • Section files: {len(section_files)}")
print(f" • Chapter files: {len(chapter_files)}")
print(f" • Other files: {len(other_files)}")
# Original smart mode logic when merging is enabled
# First, separate files with detected chapter numbers from those without
numbered_chapters = []
unnumbered_chapters = []
for idx, chapter in enumerate(candidate_chapters):
# Yield periodically during categorization (can be disabled for max speed)
if idx % 10 == 0 and idx > 0 and os.getenv("ENABLE_GUI_YIELD", "1") == "1":
time.sleep(0.001)
if chapter["num"] is not None:
numbered_chapters.append(chapter)
else:
unnumbered_chapters.append(chapter)
print(f" • Files with chapter numbers: {len(numbered_chapters)}")
print(f" • Files without chapter numbers: {len(unnumbered_chapters)}")
# Check if we have hash-based filenames (no numbered chapters found)
if not numbered_chapters and unnumbered_chapters:
print(" ⚠️ No chapter numbers found - likely hash-based filenames")
print(" → Using file order as chapter sequence")
# Sort by file index to maintain order
unnumbered_chapters.sort(key=lambda x: x["file_index"])
# Assign sequential numbers
for i, chapter in enumerate(unnumbered_chapters, 1):
chapter["num"] = i
chapter["detection_method"] = f"{extraction_mode}_hash_filename_sequential" if extraction_mode == "enhanced" else "hash_filename_sequential"
if not chapter["title"] or chapter["title"] == chapter["original_basename"]:
chapter["title"] = f"Chapter {i}"
chapters = unnumbered_chapters
else:
# We have some numbered chapters
chapters = numbered_chapters
# For unnumbered files, check if they might be duplicates or appendices
if unnumbered_chapters:
print(f" → Analyzing {len(unnumbered_chapters)} unnumbered files...")
# Get the max chapter number
max_num = max(c["num"] for c in numbered_chapters)
# Check each unnumbered file
for chapter in unnumbered_chapters:
# Check stop in post-processing loop
if is_stop_requested():
print("❌ Chapter post-processing stopped by user")
return chapters, 'unknown'
# Check if it's very small (might be a separator or note)
if chapter["file_size"] < 200:
print(f" [SKIP] Very small file: {chapter['filename']} ({chapter['file_size']} chars)")
continue
# Check if it has similar size to existing chapters (might be duplicate)
size = chapter["file_size"]
similar_chapters = [c for c in numbered_chapters
if abs(c["file_size"] - size) < 50]
if similar_chapters:
# Might be a duplicate, skip it
print(f" [SKIP] Possible duplicate: {chapter['filename']} (similar size to {len(similar_chapters)} chapters)")
continue
# Otherwise, add as appendix
max_num += 1
chapter["num"] = max_num
chapter["detection_method"] = f"{extraction_mode}_appendix_sequential" if extraction_mode == "enhanced" else "appendix_sequential"
if not chapter["title"] or chapter["title"] == chapter["original_basename"]:
chapter["title"] = f"Appendix {max_num}"
chapters.append(chapter)
print(f" [ADD] Added as chapter {max_num}: {chapter['filename']}")
else:
# For other modes or smart mode with merging disabled
chapters = chapters_direct
# Sort chapters by number
chapters.sort(key=lambda x: x["num"])
# Ensure chapter numbers are integers
# When merging is disabled, all chapters should have integer numbers anyway
for chapter in chapters:
if isinstance(chapter["num"], float):
chapter["num"] = int(chapter["num"])
# Final validation
if chapters:
print(f"\n✅ Final chapter count: {len(chapters)}")
print(f" • Chapter range: {chapters[0]['num']} - {chapters[-1]['num']}")
# Enhanced mode summary
if extraction_mode == "enhanced":
enhanced_count = sum(1 for c in chapters if c.get('enhanced_extraction', False))
print(f" 🚀 Enhanced extraction used: {enhanced_count}/{len(chapters)} chapters")
# Check for gaps
chapter_nums = [c["num"] for c in chapters]
expected_nums = list(range(min(chapter_nums), max(chapter_nums) + 1))
missing = set(expected_nums) - set(chapter_nums)
if missing:
print(f" ⚠️ Missing chapter numbers: {sorted(missing)}")
# Language detection
combined_sample = ' '.join(sample_texts) if effective_mode == "smart" else ''
detected_language = self._detect_content_language(combined_sample) if combined_sample else 'unknown'
if chapters:
self._print_extraction_summary(chapters, detected_language, extraction_mode,
h1_count if effective_mode == "smart" else 0,
h2_count if effective_mode == "smart" else 0,
file_size_groups if effective_mode == "smart" else {})
return chapters, detected_language
def _extract_chapter_info(self, soup, file_path, content_text, html_content):
"""Extract chapter number and title from various sources with parallel pattern matching"""
chapter_num = None
chapter_title = None
detection_method = None
# SPECIAL HANDLING: When we have Section/Chapter pairs, differentiate them
filename = os.path.basename(file_path)
# Handle different naming patterns for Section/Chapter files
if ('section' in filename.lower() or '_section' in filename.lower()) and 'chapter' not in filename.lower():
# For Section files, add 0.1 to the base number
# Try different patterns
match = re.search(r'No(\d+)', filename)
if not match:
match = re.search(r'^(\d+)[_\-]', filename)
if not match:
match = re.search(r'^(\d+)', filename)
if match:
base_num = int(match.group(1))
chapter_num = base_num + 0.1 # Section gets .1
detection_method = "filename_section_special"
elif ('chapter' in filename.lower() or '_chapter' in filename.lower()) and 'section' not in filename.lower():
# For Chapter files, use the base number
# Try different patterns
match = re.search(r'No(\d+)', filename)
if not match:
match = re.search(r'^(\d+)[_\-]', filename)
if not match:
match = re.search(r'^(\d+)', filename)
if match:
chapter_num = int(match.group(1))
detection_method = "filename_chapter_special"
# If not handled by special logic, continue with normal extraction
if not chapter_num:
# Try filename first - use parallel pattern matching for better performance
chapter_patterns = [(pattern, flags, method) for pattern, flags, method in self.pattern_manager.CHAPTER_PATTERNS
if method.endswith('_number')]
if len(chapter_patterns) > 3: # Only parallelize if we have enough patterns
# Parallel pattern matching for filename
with ThreadPoolExecutor(max_workers=min(4, len(chapter_patterns))) as executor:
def try_pattern(pattern_info):
pattern, flags, method = pattern_info
match = re.search(pattern, file_path, flags)
if match:
try:
num_str = match.group(1)
if num_str.isdigit():
return int(num_str), f"filename_{method}"
elif method == 'chinese_chapter_cn':
converted = self._convert_chinese_number(num_str)
if converted:
return converted, f"filename_{method}"
except (ValueError, IndexError):
pass
return None, None
# Submit all patterns
futures = [executor.submit(try_pattern, pattern_info) for pattern_info in chapter_patterns]
# Check results as they complete
for future in as_completed(futures):
try:
num, method = future.result()
if num:
chapter_num = num
detection_method = method
# Cancel remaining futures
for f in futures:
f.cancel()
break
except Exception:
continue
else:
# Sequential processing for small pattern sets
for pattern, flags, method in chapter_patterns:
match = re.search(pattern, file_path, flags)
if match:
try:
num_str = match.group(1)
if num_str.isdigit():
chapter_num = int(num_str)
detection_method = f"filename_{method}"
break
elif method == 'chinese_chapter_cn':
converted = self._convert_chinese_number(num_str)
if converted:
chapter_num = converted
detection_method = f"filename_{method}"
break
except (ValueError, IndexError):
continue
# Try content if not found in filename
if not chapter_num:
# Check ignore settings for batch translation
batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
# Prepare all text sources to check in parallel
text_sources = []
# Add title tag if not ignored
if not ignore_title_tag and soup.title and soup.title.string:
title_text = soup.title.string.strip()
text_sources.append(("title", title_text, True)) # True means this can be chapter_title
# Add headers if not ignored
if not ignore_header_tags:
for header_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
headers = soup.find_all(header_tag)
for header in headers[:3]: # Limit to first 3 of each type
header_text = header.get_text(strip=True)
if header_text:
text_sources.append((f"header_{header_tag}", header_text, True))
# Add first paragraphs
first_elements = soup.find_all(['p', 'div'])[:5]
for elem in first_elements:
elem_text = elem.get_text(strip=True)
if elem_text:
text_sources.append(("content", elem_text, False)) # False means don't use as chapter_title
# Process text sources in parallel if we have many
if len(text_sources) > 5:
with ThreadPoolExecutor(max_workers=min(6, len(text_sources))) as executor:
def extract_from_source(source_info):
source_type, text, can_be_title = source_info
num, method = self._extract_from_text(text, source_type)
return num, method, text if (num and can_be_title) else None
# Submit all text sources
future_to_source = {executor.submit(extract_from_source, source): source
for source in text_sources}
# Process results as they complete
for future in as_completed(future_to_source):
try:
num, method, title = future.result()
if num:
chapter_num = num
detection_method = method
if title and not chapter_title:
chapter_title = title
# Cancel remaining futures
for f in future_to_source:
f.cancel()
break
except Exception:
continue
else:
# Sequential processing for small text sets
for source_type, text, can_be_title in text_sources:
num, method = self._extract_from_text(text, source_type)
if num:
chapter_num = num
detection_method = method
if can_be_title and not chapter_title:
chapter_title = text
break
# Final fallback to filename patterns
if not chapter_num:
filename_base = os.path.basename(file_path)
# Parallel pattern matching for filename extraction
if len(self.pattern_manager.FILENAME_EXTRACT_PATTERNS) > 3:
with ThreadPoolExecutor(max_workers=min(4, len(self.pattern_manager.FILENAME_EXTRACT_PATTERNS))) as executor:
def try_filename_pattern(pattern):
match = re.search(pattern, filename_base, re.IGNORECASE)
if match:
try:
return int(match.group(1))
except (ValueError, IndexError):
pass
return None
futures = [executor.submit(try_filename_pattern, pattern)
for pattern in self.pattern_manager.FILENAME_EXTRACT_PATTERNS]
for future in as_completed(futures):
try:
num = future.result()
if num:
chapter_num = num
detection_method = "filename_number"
for f in futures:
f.cancel()
break
except Exception:
continue
else:
# Sequential for small pattern sets
for pattern in self.pattern_manager.FILENAME_EXTRACT_PATTERNS:
match = re.search(pattern, filename_base, re.IGNORECASE)
if match:
chapter_num = int(match.group(1))
detection_method = "filename_number"
break
# Extract title if not already found (with ignore settings support)
if not chapter_title:
# Check ignore settings for batch translation
batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
# Try title tag if not ignored
if not ignore_title_tag and soup.title and soup.title.string:
chapter_title = soup.title.string.strip()
# Try header tags if not ignored and no title found
if not chapter_title and not ignore_header_tags:
for header_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
header = soup.find(header_tag)
if header:
chapter_title = header.get_text(strip=True)
break
# Final fallback
if not chapter_title:
chapter_title = f"Chapter {chapter_num}" if chapter_num else None
chapter_title = re.sub(r'\s+', ' ', chapter_title).strip() if chapter_title else None
return chapter_num, chapter_title, detection_method
def _extract_from_text(self, text, source_type):
"""Extract chapter number from text using patterns with parallel matching for large pattern sets"""
# Get patterns that don't end with '_number'
text_patterns = [(pattern, flags, method) for pattern, flags, method in self.pattern_manager.CHAPTER_PATTERNS
if not method.endswith('_number')]
# Only use parallel processing if we have many patterns
if len(text_patterns) > 5:
with ThreadPoolExecutor(max_workers=min(4, len(text_patterns))) as executor:
def try_text_pattern(pattern_info):
pattern, flags, method = pattern_info
match = re.search(pattern, text, flags)
if match:
try:
num_str = match.group(1)
if num_str.isdigit():
return int(num_str), f"{source_type}_{method}"
elif method == 'chinese_chapter_cn':
converted = self._convert_chinese_number(num_str)
if converted:
return converted, f"{source_type}_{method}"
except (ValueError, IndexError):
pass
return None, None
# Submit all patterns
futures = [executor.submit(try_text_pattern, pattern_info) for pattern_info in text_patterns]
# Check results as they complete
for future in as_completed(futures):
try:
num, method = future.result()
if num:
# Cancel remaining futures
for f in futures:
f.cancel()
return num, method
except Exception:
continue
else:
# Sequential processing for small pattern sets
for pattern, flags, method in text_patterns:
match = re.search(pattern, text, flags)
if match:
try:
num_str = match.group(1)
if num_str.isdigit():
return int(num_str), f"{source_type}_{method}"
elif method == 'chinese_chapter_cn':
converted = self._convert_chinese_number(num_str)
if converted:
return converted, f"{source_type}_{method}"
except (ValueError, IndexError):
continue
return None, None
def _convert_chinese_number(self, cn_num):
"""Convert Chinese number to integer"""
if cn_num in self.pattern_manager.CHINESE_NUMS:
return self.pattern_manager.CHINESE_NUMS[cn_num]
if '十' in cn_num:
parts = cn_num.split('十')
if len(parts) == 2:
tens = self.pattern_manager.CHINESE_NUMS.get(parts[0], 1) if parts[0] else 1
ones = self.pattern_manager.CHINESE_NUMS.get(parts[1], 0) if parts[1] else 0
return tens * 10 + ones
return None
def _detect_content_language(self, text_sample):
"""Detect the primary language of content with parallel processing for large texts"""
# For very short texts, use sequential processing
if len(text_sample) < 1000:
scripts = {
'korean': 0,
'japanese_hiragana': 0,
'japanese_katakana': 0,
'chinese': 0,
'latin': 0
}
for char in text_sample:
code = ord(char)
if 0xAC00 <= code <= 0xD7AF:
scripts['korean'] += 1
elif 0x3040 <= code <= 0x309F:
scripts['japanese_hiragana'] += 1
elif 0x30A0 <= code <= 0x30FF:
scripts['japanese_katakana'] += 1
elif 0x4E00 <= code <= 0x9FFF:
scripts['chinese'] += 1
elif 0x0020 <= code <= 0x007F:
scripts['latin'] += 1
else:
# For longer texts, use parallel processing
# Split text into chunks for parallel processing
chunk_size = max(500, len(text_sample) // (os.cpu_count() or 4))
chunks = [text_sample[i:i + chunk_size] for i in range(0, len(text_sample), chunk_size)]
# Thread-safe accumulator
scripts_lock = threading.Lock()
scripts = {
'korean': 0,
'japanese_hiragana': 0,
'japanese_katakana': 0,
'chinese': 0,
'latin': 0
}
def process_chunk(text_chunk):
"""Process a chunk of text and return script counts"""
local_scripts = {
'korean': 0,
'japanese_hiragana': 0,
'japanese_katakana': 0,
'chinese': 0,
'latin': 0
}
for char in text_chunk:
code = ord(char)
if 0xAC00 <= code <= 0xD7AF:
local_scripts['korean'] += 1
elif 0x3040 <= code <= 0x309F:
local_scripts['japanese_hiragana'] += 1
elif 0x30A0 <= code <= 0x30FF:
local_scripts['japanese_katakana'] += 1
elif 0x4E00 <= code <= 0x9FFF:
local_scripts['chinese'] += 1
elif 0x0020 <= code <= 0x007F:
local_scripts['latin'] += 1
return local_scripts
# Process chunks in parallel
with ThreadPoolExecutor(max_workers=min(os.cpu_count() or 4, len(chunks))) as executor:
# Submit all chunks
futures = [executor.submit(process_chunk, chunk) for chunk in chunks]
# Collect results
for future in as_completed(futures):
try:
chunk_scripts = future.result()
# Thread-safe accumulation
with scripts_lock:
for script, count in chunk_scripts.items():
scripts[script] += count
except Exception as e:
print(f"[WARNING] Error processing chunk in language detection: {e}")
# Language determination logic (same as original)
total_cjk = scripts['korean'] + scripts['japanese_hiragana'] + scripts['japanese_katakana'] + scripts['chinese']
if scripts['korean'] > total_cjk * 0.3:
return 'korean'
elif scripts['japanese_hiragana'] + scripts['japanese_katakana'] > total_cjk * 0.2:
return 'japanese'
elif scripts['chinese'] > total_cjk * 0.3:
return 'chinese'
elif scripts['latin'] > len(text_sample) * 0.7:
return 'english'
else:
return 'unknown'
def _print_extraction_summary(self, chapters, detected_language, extraction_mode, h1_count, h2_count, file_size_groups):
"""Print extraction summary"""
print(f"\n📊 Chapter Extraction Summary ({extraction_mode.capitalize()} Mode):")
print(f" • Total chapters extracted: {len(chapters)}")
# Format chapter range handling both int and float
first_num = chapters[0]['num']
last_num = chapters[-1]['num']
print(f" • Chapter range: {first_num} to {last_num}")
print(f" • Detected language: {detected_language}")
if extraction_mode == "smart":
print(f" • Primary header type: {'
' if h2_count > h1_count else ''}")
image_only_count = sum(1 for c in chapters if c.get('is_image_only', False))
text_only_count = sum(1 for c in chapters if not c.get('has_images', False) and c.get('file_size', 0) >= 500)
mixed_count = sum(1 for c in chapters if c.get('has_images', False) and c.get('file_size', 0) >= 500)
empty_count = sum(1 for c in chapters if c.get('file_size', 0) < 50)
print(f" • Text-only chapters: {text_only_count}")
print(f" • Image-only chapters: {image_only_count}")
print(f" • Mixed content chapters: {mixed_count}")
print(f" • Empty/minimal content: {empty_count}")
# Check for merged chapters
merged_count = sum(1 for c in chapters if c.get('was_merged', False))
if merged_count > 0:
print(f" • Merged chapters: {merged_count}")
# Check for missing chapters (only for integer sequences)
expected_chapters = set(range(chapters[0]['num'], chapters[-1]['num'] + 1))
actual_chapters = set(c['num'] for c in chapters)
missing = expected_chapters - actual_chapters
if missing:
print(f" ⚠️ Missing chapter numbers: {sorted(missing)}")
if extraction_mode == "smart":
method_stats = Counter(c['detection_method'] for c in chapters)
print(f" 📈 Detection methods used:")
for method, count in method_stats.most_common():
print(f" • {method}: {count} chapters")
large_groups = [size for size, files in file_size_groups.items() if len(files) > 1]
if large_groups:
print(f" ⚠️ Found {len(large_groups)} file size groups with potential duplicates")
else:
print(f" • Empty/placeholder: {empty_count}")
if extraction_mode == "full":
print(f" 🔍 Full extraction preserved all HTML structure and tags")
def _extract_epub_metadata(self, zf):
"""Extract comprehensive metadata from EPUB file including all custom fields"""
meta = {}
# Use lxml for XML if available
xml_parser = 'lxml-xml' if self.parser == 'lxml' else 'xml'
try:
for name in zf.namelist():
if name.lower().endswith('.opf'):
opf_content = zf.read(name)
soup = BeautifulSoup(opf_content, xml_parser)
# Extract ALL Dublin Core elements (expanded list)
dc_elements = ['title', 'creator', 'subject', 'description',
'publisher', 'contributor', 'date', 'type',
'format', 'identifier', 'source', 'language',
'relation', 'coverage', 'rights']
for element in dc_elements:
tag = soup.find(element)
if tag and tag.get_text(strip=True):
meta[element] = tag.get_text(strip=True)
# Extract ALL meta tags (not just series)
meta_tags = soup.find_all('meta')
for meta_tag in meta_tags:
# Try different attribute names for the metadata name
name = meta_tag.get('name') or meta_tag.get('property', '')
content = meta_tag.get('content', '')
if name and content:
# Store original name for debugging
original_name = name
# Clean up common prefixes
if name.startswith('calibre:'):
name = name[8:] # Remove 'calibre:' prefix
elif name.startswith('dc:'):
name = name[3:] # Remove 'dc:' prefix
elif name.startswith('opf:'):
name = name[4:] # Remove 'opf:' prefix
# Normalize the field name - replace hyphens with underscores
name = name.replace('-', '_')
# Don't overwrite if already exists (prefer direct tags over meta tags)
if name not in meta:
meta[name] = content
# Debug output for custom fields
if original_name != name:
print(f" • Found custom field: {original_name} → {name}")
# Special handling for series information (maintain compatibility)
if 'series' not in meta:
series_tags = soup.find_all('meta', attrs={'name': lambda x: x and 'series' in x.lower()})
for series_tag in series_tags:
series_name = series_tag.get('content', '')
if series_name:
meta['series'] = series_name
break
# Extract refines metadata (used by some EPUB creators)
refines_metas = soup.find_all('meta', attrs={'refines': True})
for refine in refines_metas:
property_name = refine.get('property', '')
content = refine.get_text(strip=True) or refine.get('content', '')
if property_name and content:
# Clean property name
if ':' in property_name:
property_name = property_name.split(':')[-1]
property_name = property_name.replace('-', '_')
if property_name not in meta:
meta[property_name] = content
# Log extraction summary
print(f"📋 Extracted {len(meta)} metadata fields")
# Show standard vs custom fields
standard_keys = {'title', 'creator', 'language', 'subject', 'description',
'publisher', 'date', 'identifier', 'source', 'rights',
'contributor', 'type', 'format', 'relation', 'coverage'}
custom_keys = set(meta.keys()) - standard_keys
if custom_keys:
print(f"📋 Standard fields: {len(standard_keys & set(meta.keys()))}")
print(f"📋 Custom fields found: {sorted(custom_keys)}")
# Show sample values for custom fields (truncated)
for key in sorted(custom_keys)[:5]: # Show first 5 custom fields
value = str(meta[key])
if len(value) > 50:
value = value[:47] + "..."
print(f" • {key}: {value}")
if len(custom_keys) > 5:
print(f" • ... and {len(custom_keys) - 5} more custom fields")
break
except Exception as e:
print(f"[WARNING] Failed to extract metadata: {e}")
import traceback
traceback.print_exc()
return meta
def _categorize_resource(self, file_path, file_name):
"""Categorize a file and return (resource_type, target_dir, safe_filename)"""
file_path_lower = file_path.lower()
file_name_lower = file_name.lower()
if file_path_lower.endswith('.css'):
return 'css', 'css', sanitize_resource_filename(file_name)
elif file_path_lower.endswith(('.ttf', '.otf', '.woff', '.woff2', '.eot')):
return 'fonts', 'fonts', sanitize_resource_filename(file_name)
elif file_path_lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.svg', '.bmp', '.webp')):
return 'images', 'images', sanitize_resource_filename(file_name)
elif (file_path_lower.endswith(('.opf', '.ncx')) or
file_name_lower == 'container.xml' or
'container.xml' in file_path_lower):
if 'container.xml' in file_path_lower:
safe_filename = 'container.xml'
else:
safe_filename = file_name
return 'epub_structure', None, safe_filename
elif file_path_lower.endswith(('.js', '.xml', '.txt')):
return 'other', None, sanitize_resource_filename(file_name)
return None
def _cleanup_old_resources(self, output_dir):
"""Clean up old resource directories and EPUB structure files"""
print("🧹 Cleaning up any existing resource directories...")
cleanup_success = True
for resource_type in ['css', 'fonts', 'images']:
resource_dir = os.path.join(output_dir, resource_type)
if os.path.exists(resource_dir):
try:
shutil.rmtree(resource_dir)
print(f" 🗑️ Removed old {resource_type} directory")
except PermissionError as e:
print(f" ⚠️ Cannot remove {resource_type} directory (permission denied) - will merge with existing files")
cleanup_success = False
except Exception as e:
print(f" ⚠️ Error removing {resource_type} directory: {e} - will merge with existing files")
cleanup_success = False
epub_structure_files = ['container.xml', 'content.opf', 'toc.ncx']
for epub_file in epub_structure_files:
input_path = os.path.join(output_dir, epub_file)
if os.path.exists(input_path):
try:
os.remove(input_path)
print(f" 🗑️ Removed old {epub_file}")
except PermissionError:
print(f" ⚠️ Cannot remove {epub_file} (permission denied) - will use existing file")
except Exception as e:
print(f" ⚠️ Error removing {epub_file}: {e}")
try:
for file in os.listdir(output_dir):
if file.lower().endswith(('.opf', '.ncx')):
file_path = os.path.join(output_dir, file)
try:
os.remove(file_path)
print(f" 🗑️ Removed old EPUB file: {file}")
except PermissionError:
print(f" ⚠️ Cannot remove {file} (permission denied)")
except Exception as e:
print(f" ⚠️ Error removing {file}: {e}")
except Exception as e:
print(f"⚠️ Error scanning for EPUB files: {e}")
if not cleanup_success:
print("⚠️ Some cleanup operations failed due to file permissions")
print(" The program will continue and merge with existing files")
return cleanup_success
def _count_existing_resources(self, output_dir, extracted_resources):
"""Count existing resources when skipping extraction"""
for resource_type in ['css', 'fonts', 'images', 'epub_structure']:
if resource_type == 'epub_structure':
epub_files = []
for file in ['container.xml', 'content.opf', 'toc.ncx']:
if os.path.exists(os.path.join(output_dir, file)):
epub_files.append(file)
try:
for file in os.listdir(output_dir):
if file.lower().endswith(('.opf', '.ncx')) and file not in epub_files:
epub_files.append(file)
except:
pass
extracted_resources[resource_type] = epub_files
else:
resource_dir = os.path.join(output_dir, resource_type)
if os.path.exists(resource_dir):
try:
files = [f for f in os.listdir(resource_dir) if os.path.isfile(os.path.join(resource_dir, f))]
extracted_resources[resource_type] = files
except:
extracted_resources[resource_type] = []
total_existing = sum(len(files) for files in extracted_resources.values())
print(f"✅ Found {total_existing} existing resource files")
return extracted_resources
def _validate_critical_files(self, output_dir, extracted_resources):
"""Validate that critical EPUB files were extracted"""
total_extracted = sum(len(files) for files in extracted_resources.values())
print(f"✅ Extracted {total_extracted} resource files:")
for resource_type, files in extracted_resources.items():
if files:
if resource_type == 'epub_structure':
print(f" • EPUB Structure: {len(files)} files")
for file in files:
print(f" - {file}")
else:
print(f" • {resource_type.title()}: {len(files)} files")
critical_files = ['container.xml']
missing_critical = [f for f in critical_files if not os.path.exists(os.path.join(output_dir, f))]
if missing_critical:
print(f"⚠️ WARNING: Missing critical EPUB files: {missing_critical}")
print(" This may prevent proper EPUB reconstruction!")
else:
print("✅ All critical EPUB structure files extracted successfully")
opf_files = [f for f in extracted_resources['epub_structure'] if f.lower().endswith('.opf')]
if not opf_files:
print("⚠️ WARNING: No OPF file found! This will prevent EPUB reconstruction.")
else:
print(f"✅ Found OPF file(s): {opf_files}")
def _create_extraction_report(self, output_dir, metadata, chapters, extracted_resources):
"""Create comprehensive extraction report with HTML file tracking"""
report_path = os.path.join(output_dir, 'extraction_report.txt')
with open(report_path, 'w', encoding='utf-8') as f:
f.write("EPUB Extraction Report\n")
f.write("=" * 50 + "\n\n")
f.write(f"EXTRACTION MODE: {metadata.get('extraction_mode', 'unknown').upper()}\n\n")
f.write("METADATA:\n")
for key, value in metadata.items():
if key not in ['chapter_titles', 'extracted_resources', 'extraction_mode']:
f.write(f" {key}: {value}\n")
f.write(f"\nCHAPTERS ({len(chapters)}):\n")
text_chapters = []
image_only_chapters = []
mixed_chapters = []
for chapter in chapters:
if chapter.get('has_images') and chapter.get('file_size', 0) < 500:
image_only_chapters.append(chapter)
elif chapter.get('has_images') and chapter.get('file_size', 0) >= 500:
mixed_chapters.append(chapter)
else:
text_chapters.append(chapter)
if text_chapters:
f.write(f"\n TEXT CHAPTERS ({len(text_chapters)}):\n")
for c in text_chapters:
f.write(f" {c['num']:3d}. {c['title']} ({c['detection_method']})\n")
if c.get('original_html_file'):
f.write(f" → {c['original_html_file']}\n")
if image_only_chapters:
f.write(f"\n IMAGE-ONLY CHAPTERS ({len(image_only_chapters)}):\n")
for c in image_only_chapters:
f.write(f" {c['num']:3d}. {c['title']} (images: {c.get('image_count', 0)})\n")
if c.get('original_html_file'):
f.write(f" → {c['original_html_file']}\n")
if 'body' in c:
try:
soup = BeautifulSoup(c['body'], 'html.parser')
images = soup.find_all('img')
for img in images[:3]:
src = img.get('src', 'unknown')
f.write(f" • Image: {src}\n")
if len(images) > 3:
f.write(f" • ... and {len(images) - 3} more images\n")
except:
pass
if mixed_chapters:
f.write(f"\n MIXED CONTENT CHAPTERS ({len(mixed_chapters)}):\n")
for c in mixed_chapters:
f.write(f" {c['num']:3d}. {c['title']} (text: {c.get('file_size', 0)} chars, images: {c.get('image_count', 0)})\n")
if c.get('original_html_file'):
f.write(f" → {c['original_html_file']}\n")
f.write(f"\nRESOURCES EXTRACTED:\n")
for resource_type, files in extracted_resources.items():
if files:
if resource_type == 'epub_structure':
f.write(f" EPUB Structure: {len(files)} files\n")
for file in files:
f.write(f" - {file}\n")
else:
f.write(f" {resource_type.title()}: {len(files)} files\n")
for file in files[:5]:
f.write(f" - {file}\n")
if len(files) > 5:
f.write(f" ... and {len(files) - 5} more\n")
f.write(f"\nHTML FILES WRITTEN:\n")
html_files_written = metadata.get('html_files_written', 0)
f.write(f" Total: {html_files_written} files\n")
f.write(f" Location: Main directory and 'originals' subdirectory\n")
f.write(f"\nPOTENTIAL ISSUES:\n")
issues = []
if image_only_chapters:
issues.append(f" • {len(image_only_chapters)} chapters contain only images (may need OCR)")
missing_html = sum(1 for c in chapters if not c.get('original_html_file'))
if missing_html > 0:
issues.append(f" • {missing_html} chapters failed to write HTML files")
if not extracted_resources.get('epub_structure'):
issues.append(" • No EPUB structure files found (may affect reconstruction)")
if not issues:
f.write(" None detected - extraction appears successful!\n")
else:
for issue in issues:
f.write(issue + "\n")
print(f"📄 Saved extraction report to: {report_path}")
def _log_extraction_summary(self, chapters, extracted_resources, detected_language, html_files_written=0):
"""Log final extraction summary with HTML file information"""
extraction_mode = chapters[0].get('extraction_mode', 'unknown') if chapters else 'unknown'
print(f"\n✅ {extraction_mode.capitalize()} extraction complete!")
print(f" 📚 Chapters: {len(chapters)}")
print(f" 📄 HTML files written: {html_files_written}")
print(f" 🎨 Resources: {sum(len(files) for files in extracted_resources.values())}")
print(f" 🌍 Language: {detected_language}")
image_only_count = sum(1 for c in chapters if c.get('has_images') and c.get('file_size', 0) < 500)
if image_only_count > 0:
print(f" 📸 Image-only chapters: {image_only_count}")
epub_files = extracted_resources.get('epub_structure', [])
if epub_files:
print(f" 📋 EPUB Structure: {len(epub_files)} files ({', '.join(epub_files)})")
else:
print(f" ⚠️ No EPUB structure files extracted!")
print(f"\n🔍 Pre-flight check readiness:")
print(f" ✅ HTML files: {'READY' if html_files_written > 0 else 'NOT READY'}")
print(f" ✅ Metadata: READY")
print(f" ✅ Resources: READY")
# =====================================================
# UNIFIED TRANSLATION PROCESSOR
# =====================================================
class TranslationProcessor:
"""Handles the translation of individual chapters"""
def __init__(self, config, client, out_dir, log_callback=None, stop_callback=None, uses_zero_based=False, is_text_file=False):
self.config = config
self.client = client
self.out_dir = out_dir
self.log_callback = log_callback
self.stop_callback = stop_callback
self.chapter_splitter = ChapterSplitter(model_name=config.MODEL)
self.uses_zero_based = uses_zero_based
self.is_text_file = is_text_file
# Check and log multi-key status
if hasattr(self.client, 'use_multi_keys') and self.client.use_multi_keys:
stats = self.client.get_stats()
self._log(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys")
self._log(f" Active keys: {stats.get('active_keys', 0)}")
def _log(self, message):
"""Log a message"""
if self.log_callback:
self.log_callback(message)
else:
print(message)
def report_key_status(self):
"""Report multi-key status if available"""
if hasattr(self.client, 'get_stats'):
stats = self.client.get_stats()
if stats.get('multi_key_mode', False):
self._log(f"\n📊 API Key Status:")
self._log(f" Active Keys: {stats.get('active_keys', 0)}/{stats.get('total_keys', 0)}")
self._log(f" Success Rate: {stats.get('success_rate', 0):.1%}")
self._log(f" Total Requests: {stats.get('total_requests', 0)}\n")
def check_stop(self):
"""Check if translation should stop"""
if self.stop_callback and self.stop_callback():
print("❌ Translation stopped by user request.")
return True
def check_duplicate_content(self, result, idx, prog, out, actual_num=None):
"""Check if translated content is duplicate - with mode selection"""
# Get detection mode from config
detection_mode = getattr(self.config, 'DUPLICATE_DETECTION_MODE', 'basic')
print(f" 🔍 DEBUG: Detection mode = '{detection_mode}'")
print(f" 🔍 DEBUG: Lookback chapters = {self.config.DUPLICATE_LOOKBACK_CHAPTERS}")
# Extract content_hash if available from progress
content_hash = None
if detection_mode == 'ai-hunter':
# Try to get content_hash from the current chapter info
# Use actual_num if provided, otherwise fallback to idx+1
if actual_num is not None:
chapter_key = str(actual_num)
else:
chapter_key = str(idx + 1)
if chapter_key in prog.get("chapters", {}):
chapter_info = prog["chapters"][chapter_key]
content_hash = chapter_info.get("content_hash")
print(f" 🔍 DEBUG: Found content_hash for chapter {idx}: {content_hash}")
if detection_mode == 'ai-hunter':
print(" 🤖 DEBUG: Routing to AI Hunter detection...")
# Check if AI Hunter method is available (injected by the wrapper)
if hasattr(self, '_check_duplicate_ai_hunter'):
return self._check_duplicate_ai_hunter(result, idx, prog, out, content_hash)
else:
print(" ⚠️ AI Hunter method not available, falling back to basic detection")
return self._check_duplicate_basic(result, idx, prog, out)
elif detection_mode == 'cascading':
print(" 🔄 DEBUG: Routing to Cascading detection...")
return self._check_duplicate_cascading(result, idx, prog, out)
else:
print(" 📋 DEBUG: Routing to Basic detection...")
return self._check_duplicate_basic(result, idx, prog, out)
def _check_duplicate_basic(self, result, idx, prog, out):
"""Original basic duplicate detection"""
try:
result_clean = re.sub(r'<[^>]+>', '', result).strip().lower()
result_sample = result_clean[:1000]
lookback_chapters = self.config.DUPLICATE_LOOKBACK_CHAPTERS
for prev_idx in range(max(0, idx - lookback_chapters), idx):
prev_key = str(prev_idx)
if prev_key in prog["chapters"] and prog["chapters"][prev_key].get("output_file"):
prev_file = prog["chapters"][prev_key]["output_file"]
prev_path = os.path.join(out, prev_file)
if os.path.exists(prev_path):
try:
with open(prev_path, 'r', encoding='utf-8') as f:
prev_content = f.read()
prev_clean = re.sub(r'<[^>]+>', '', prev_content).strip().lower()
prev_sample = prev_clean[:1000]
# Use SequenceMatcher for similarity comparison
similarity = SequenceMatcher(None, result_sample, prev_sample).ratio()
if similarity >= 0.85: # 85% threshold
print(f" 🚀 Basic detection: Duplicate found ({int(similarity*100)}%)")
return True, int(similarity * 100)
except Exception as e:
print(f" Warning: Failed to read {prev_path}: {e}")
continue
return False, 0
except Exception as e:
print(f" Warning: Failed to check duplicate content: {e}")
return False, 0
def _check_duplicate_cascading(self, result, idx, prog, out):
"""Cascading detection - basic first, then AI Hunter for borderline cases"""
# Step 1: Basic
is_duplicate_basic, similarity_basic = self._check_duplicate_basic(result, idx, prog, out)
if is_duplicate_basic:
return True, similarity_basic
# Step 2: If basic detection finds moderate similarity, use AI Hunter
if similarity_basic >= 60: # Configurable threshold
print(f" 🤖 Moderate similarity ({similarity_basic}%) - running AI Hunter analysis...")
if hasattr(self, '_check_duplicate_ai_hunter'):
is_duplicate_ai, similarity_ai = self._check_duplicate_ai_hunter(result, idx, prog, out)
if is_duplicate_ai:
return True, similarity_ai
else:
print(" ⚠️ AI Hunter method not available for cascading analysis")
return False, max(similarity_basic, 0)
def _extract_text_features(self, text):
"""Extract multiple features from text for AI Hunter analysis"""
features = {
'semantic': {},
'structural': {},
'characters': [],
'patterns': {}
}
# Semantic fingerprint
lines = text.split('\n')
# Character extraction (names that appear 3+ times)
words = re.findall(r'\b[A-Z][a-z]+\b', text)
word_freq = Counter(words)
features['characters'] = [name for name, count in word_freq.items() if count >= 3]
# Dialogue patterns
dialogue_patterns = re.findall(r'"([^"]+)"', text)
features['semantic']['dialogue_count'] = len(dialogue_patterns)
features['semantic']['dialogue_lengths'] = [len(d) for d in dialogue_patterns[:10]]
# Speaker patterns
speaker_patterns = re.findall(r'(\w+)\s+(?:said|asked|replied|shouted|whispered)', text.lower())
features['semantic']['speakers'] = list(set(speaker_patterns[:20]))
# Number extraction
numbers = re.findall(r'\b\d+\b', text)
features['patterns']['numbers'] = numbers[:20]
# Structural signature
para_lengths = []
dialogue_count = 0
for para in text.split('\n\n'):
if para.strip():
para_lengths.append(len(para))
if '"' in para:
dialogue_count += 1
features['structural']['para_count'] = len(para_lengths)
features['structural']['avg_para_length'] = sum(para_lengths) / max(1, len(para_lengths))
features['structural']['dialogue_ratio'] = dialogue_count / max(1, len(para_lengths))
# Create structural pattern string
pattern = []
for para in text.split('\n\n')[:20]: # First 20 paragraphs
if para.strip():
if '"' in para:
pattern.append('D') # Dialogue
elif len(para) > 300:
pattern.append('L') # Long
elif len(para) < 100:
pattern.append('S') # Short
else:
pattern.append('M') # Medium
features['structural']['pattern'] = ''.join(pattern)
return features
def _calculate_exact_similarity(self, text1, text2):
"""Calculate exact text similarity"""
return SequenceMatcher(None, text1.lower(), text2.lower()).ratio()
def _calculate_smart_similarity(self, text1, text2):
"""Smart similarity with length-aware sampling"""
# Check length ratio first
len_ratio = len(text1) / max(1, len(text2))
if len_ratio < 0.7 or len_ratio > 1.3:
return 0.0
# Smart sampling for large texts
if len(text1) > 10000:
sample_size = 3000
samples1 = [
text1[:sample_size],
text1[len(text1)//2 - sample_size//2:len(text1)//2 + sample_size//2],
text1[-sample_size:]
]
samples2 = [
text2[:sample_size],
text2[len(text2)//2 - sample_size//2:len(text2)//2 + sample_size//2],
text2[-sample_size:]
]
similarities = [SequenceMatcher(None, s1.lower(), s2.lower()).ratio()
for s1, s2 in zip(samples1, samples2)]
return sum(similarities) / len(similarities)
else:
# Use first 2000 chars for smaller texts
return SequenceMatcher(None, text1[:2000].lower(), text2[:2000].lower()).ratio()
def _calculate_semantic_similarity(self, sem1, sem2):
"""Calculate semantic fingerprint similarity"""
score = 0.0
max_score = 0.0
# Compare dialogue counts
if 'dialogue_count' in sem1 and 'dialogue_count' in sem2:
max_score += 1.0
ratio = min(sem1['dialogue_count'], sem2['dialogue_count']) / max(1, max(sem1['dialogue_count'], sem2['dialogue_count']))
score += ratio * 0.3
# Compare speakers
if 'speakers' in sem1 and 'speakers' in sem2:
max_score += 1.0
if sem1['speakers'] and sem2['speakers']:
overlap = len(set(sem1['speakers']) & set(sem2['speakers']))
total = len(set(sem1['speakers']) | set(sem2['speakers']))
score += (overlap / max(1, total)) * 0.4
# Compare dialogue lengths pattern
if 'dialogue_lengths' in sem1 and 'dialogue_lengths' in sem2:
max_score += 1.0
if sem1['dialogue_lengths'] and sem2['dialogue_lengths']:
# Compare dialogue length patterns
len1 = sem1['dialogue_lengths'][:10]
len2 = sem2['dialogue_lengths'][:10]
if len1 and len2:
avg1 = sum(len1) / len(len1)
avg2 = sum(len2) / len(len2)
ratio = min(avg1, avg2) / max(1, max(avg1, avg2))
score += ratio * 0.3
return score / max(1, max_score)
def _calculate_structural_similarity(self, struct1, struct2):
"""Calculate structural signature similarity"""
score = 0.0
# Compare paragraph patterns
if 'pattern' in struct1 and 'pattern' in struct2:
pattern_sim = SequenceMatcher(None, struct1['pattern'], struct2['pattern']).ratio()
score += pattern_sim * 0.4
# Compare paragraph statistics
if all(k in struct1 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']) and \
all(k in struct2 for k in ['para_count', 'avg_para_length', 'dialogue_ratio']):
# Paragraph count ratio
para_ratio = min(struct1['para_count'], struct2['para_count']) / max(1, max(struct1['para_count'], struct2['para_count']))
score += para_ratio * 0.2
# Average length ratio
avg_ratio = min(struct1['avg_para_length'], struct2['avg_para_length']) / max(1, max(struct1['avg_para_length'], struct2['avg_para_length']))
score += avg_ratio * 0.2
# Dialogue ratio similarity
dialogue_diff = abs(struct1['dialogue_ratio'] - struct2['dialogue_ratio'])
score += (1 - dialogue_diff) * 0.2
return score
def _calculate_character_similarity(self, chars1, chars2):
"""Calculate character name similarity"""
if not chars1 or not chars2:
return 0.0
# Find overlapping characters
set1 = set(chars1)
set2 = set(chars2)
overlap = len(set1 & set2)
total = len(set1 | set2)
return overlap / max(1, total)
def _calculate_pattern_similarity(self, pat1, pat2):
"""Calculate pattern-based similarity"""
score = 0.0
# Compare numbers (they rarely change in translations)
if 'numbers' in pat1 and 'numbers' in pat2:
nums1 = set(pat1['numbers'])
nums2 = set(pat2['numbers'])
if nums1 and nums2:
overlap = len(nums1 & nums2)
total = len(nums1 | nums2)
score = overlap / max(1, total)
return score
def generate_rolling_summary(self, history_manager, chapter_num, base_system_content=None, source_text=None):
"""Generate rolling summary after a chapter for context continuity.
Uses a dedicated summary system prompt (with glossary) distinct from translation.
Writes the summary to rolling_summary.txt and returns the summary string.
"""
if not self.config.USE_ROLLING_SUMMARY:
return None
current_history = history_manager.load_history()
messages_to_include = self.config.ROLLING_SUMMARY_EXCHANGES * 2
# Prefer directly provided source text (e.g., just-translated chapter) when available
assistant_responses = []
if source_text and isinstance(source_text, str) and source_text.strip():
assistant_responses = [source_text]
else:
if len(current_history) >= 2:
recent_messages = current_history[-messages_to_include:] if messages_to_include > 0 else current_history
for h in recent_messages:
if h.get("role") == "assistant":
assistant_responses.append(h["content"])
# If still empty, skip quietly
if not assistant_responses:
return None
# Build a dedicated summary system prompt (do NOT reuse main translation system prompt)
# Append glossary to keep terminology consistent
summary_system_template = os.getenv("ROLLING_SUMMARY_SYSTEM_PROMPT", "You create concise summaries for continuity.").strip()
try:
glossary_path = find_glossary_file(self.out_dir)
except Exception:
glossary_path = None
system_prompt = build_system_prompt(summary_system_template, glossary_path)
# Add explicit instruction for clarity
system_prompt += "\n\n[Instruction: Generate a concise rolling summary of the previous chapter. Use glossary terms consistently. Do not include warnings or explanations.]"
user_prompt_template = os.getenv(
"ROLLING_SUMMARY_USER_PROMPT",
"Summarize the key events, characters, tone, and important details from these translations. "
"Focus on: character names/relationships, plot developments, and any special terminology used.\n\n"
"{translations}"
)
translations_text = "\n---\n".join(assistant_responses)
user_prompt = user_prompt_template.replace("{translations}", translations_text)
summary_msgs = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"[Rolling Summary of Chapter {chapter_num}]\n" + user_prompt}
]
try:
summary_resp, _ = send_with_interrupt(
summary_msgs, self.client, self.config.TEMP,
min(2000, self.config.MAX_OUTPUT_TOKENS),
self.check_stop,
context='summary'
)
# Save the summary to the output folder
summary_file = os.path.join(self.out_dir, "rolling_summary.txt")
header = f"=== Rolling Summary of Chapter {chapter_num} ===\n(This is a summary of the previous chapter for context)\n"
mode = "a" if self.config.ROLLING_SUMMARY_MODE == "append" else "w"
with open(summary_file, mode, encoding="utf-8") as sf:
if mode == "a":
sf.write("\n\n")
sf.write(header)
sf.write(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}]\n")
sf.write(summary_resp.strip())
# If in append mode, trim to retain only the last N entries if configured
try:
if self.config.ROLLING_SUMMARY_MODE == "append":
max_entries = int(getattr(self.config, "ROLLING_SUMMARY_MAX_ENTRIES", 0) or 0)
if max_entries > 0:
with open(summary_file, 'r', encoding='utf-8') as rf:
content = rf.read()
# Find the start of each summary block by header line
headers = [m.start() for m in re.finditer(r"(?m)^===\s*Rolling Summary.*$", content)]
if len(headers) > max_entries:
# Keep only the last max_entries blocks
keep_starts = headers[-max_entries:]
blocks = []
for i, s in enumerate(keep_starts):
e = keep_starts[i + 1] if i + 1 < len(keep_starts) else len(content)
block = content[s:e].strip()
if block:
blocks.append(block)
trimmed_content = ("\n\n".join(blocks) + "\n") if blocks else ""
with open(summary_file, 'w', encoding='utf-8') as wf:
wf.write(trimmed_content)
# Optional log showing retained count
try:
self._log(f"📚 Total summaries in memory: {len(blocks)} (trimmed to last {max_entries})")
except Exception:
pass
except Exception as _trim_err:
try:
self._log(f"⚠️ Failed to trim rolling summaries: {_trim_err}")
except Exception:
pass
# Log to GUI if available, otherwise console
try:
self._log(f"📝 Generated rolling summary for Chapter {chapter_num} ({'append' if mode=='a' else 'replace'} mode)")
self._log(f" ➜ Saved to: {summary_file} ({len(summary_resp.strip())} chars)")
except Exception:
print(f"📝 Generated rolling summary for Chapter {chapter_num} ({'append' if mode=='a' else 'replace'} mode)")
print(f" ➜ Saved to: {summary_file} ({len(summary_resp.strip())} chars)")
return summary_resp.strip()
except Exception as e:
try:
self._log(f"⚠️ Failed to generate rolling summary: {e}")
except Exception:
print(f"⚠️ Failed to generate rolling summary: {e}")
return None
def translate_with_retry(self, msgs, chunk_html, c, chunk_idx, total_chunks):
"""Handle translation with retry logic"""
# CRITICAL FIX: Reset client state for each chunk
if hasattr(self.client, 'reset_cleanup_state'):
self.client.reset_cleanup_state()
# Also ensure we're not in cleanup mode from previous operations
if hasattr(self.client, '_in_cleanup'):
self.client._in_cleanup = False
if hasattr(self.client, '_cancelled'):
self.client._cancelled = False
retry_count = 0
# Get retry attempts from AI Hunter config if available
ai_config = {}
try:
# Try to get AI Hunter config from environment variable first
ai_hunter_config_str = os.getenv('AI_HUNTER_CONFIG')
if ai_hunter_config_str:
ai_config = json.loads(ai_hunter_config_str)
else:
# Fallback to config attribute
ai_config = getattr(self.config, 'ai_hunter_config', {})
except (json.JSONDecodeError, AttributeError):
ai_config = {}
if isinstance(ai_config, dict):
max_retries = ai_config.get('retry_attempts', 3)
max_duplicate_retries = ai_config.get('retry_attempts', 6) # Use same setting for duplicate retries
else:
max_retries = 3
max_duplicate_retries = 6
duplicate_retry_count = 0
timeout_retry_count = 0
max_timeout_retries = 2
history_purged = False
original_max_tokens = self.config.MAX_OUTPUT_TOKENS
original_temp = self.config.TEMP
original_user_prompt = msgs[-1]["content"]
chunk_timeout = None
if self.config.RETRY_TIMEOUT:
chunk_timeout = self.config.CHUNK_TIMEOUT
result = None
finish_reason = None
while True:
if self.check_stop():
return None, None
try:
current_max_tokens = self.config.MAX_OUTPUT_TOKENS
current_temp = self.config.TEMP
total_tokens = sum(self.chapter_splitter.count_tokens(m["content"]) for m in msgs)
# Determine file reference
if c.get('is_chunk', False):
file_ref = f"Section_{c['num']}"
else:
# Check if this is a text file - need to access from self
is_text_source = self.is_text_file or c.get('filename', '').endswith('.txt')
terminology = "Section" if is_text_source else "Chapter"
file_ref = c.get('original_basename', f'{terminology}_{c["num"]}')
print(f"[DEBUG] Chunk {chunk_idx}/{total_chunks} tokens = {total_tokens:,} / {self.get_token_budget_str()} [File: {file_ref}]")
self.client.context = 'translation'
# Generate filename for chunks
if chunk_idx and total_chunks > 1:
# This is a chunk - use chunk naming format
fname = f"response_{c['num']:03d}_chunk_{chunk_idx}.html"
else:
# Not a chunk - use regular naming
fname = FileUtilities.create_chapter_filename(c, c.get('actual_chapter_num', c['num']))
# Set output filename BEFORE the API call
if hasattr(self.client, 'set_output_filename'):
self.client.set_output_filename(fname)
# Track the filename so truncation logs know which file this is
if hasattr(self.client, '_current_output_file'):
self.client._current_output_file = fname
# Generate unique request ID for this chunk
#request_id = f"{c['num']:03d}_chunk{chunk_idx}_{uuid.uuid4().hex[:8]}"
result, finish_reason = send_with_interrupt(
msgs, self.client, current_temp, current_max_tokens,
self.check_stop, chunk_timeout
)
# Enhanced mode workflow:
# 1. Original HTML -> html2text -> Markdown/plain text (during extraction)
# 2. Markdown sent to translation API (better for translation quality)
# 3. Translated markdown -> HTML conversion (here)
if result and c.get("enhanced_extraction", False):
print(f"🔄 Converting translated markdown back to HTML...")
result = convert_enhanced_text_to_html(result, c)
retry_needed = False
retry_reason = ""
is_duplicate_retry = False
# ENHANCED: Force re-read environment variable for latest setting
retry_truncated_enabled = os.getenv("RETRY_TRUNCATED", "0") == "1"
# Debug logging to verify the toggle state
#print(f" DEBUG: finish_reason='{finish_reason}', RETRY_TRUNCATED={retry_truncated_enabled}, config.RETRY_TRUNCATED={self.config.RETRY_TRUNCATED}")
#print(f" DEBUG: Current tokens={self.config.MAX_OUTPUT_TOKENS}, Min retry tokens={self.config.MAX_RETRY_TOKENS}, retry_count={retry_count}")
if finish_reason == "length" and (retry_truncated_enabled or self.config.RETRY_TRUNCATED):
if retry_count < max_retries:
# For truncated responses, ensure we never go below the minimum retry tokens
proposed_limit = self.config.MAX_OUTPUT_TOKENS * 2
# Always enforce minimum - never retry with tokens below the constraint
new_token_limit = max(proposed_limit, self.config.MAX_RETRY_TOKENS)
if new_token_limit != self.config.MAX_OUTPUT_TOKENS:
retry_needed = True
retry_reason = "truncated output"
old_limit = self.config.MAX_OUTPUT_TOKENS
self.config.MAX_OUTPUT_TOKENS = new_token_limit
retry_count += 1
if old_limit < self.config.MAX_RETRY_TOKENS:
print(f" 🔄 TRUNCATION RETRY: Boosting tokens {old_limit} → {new_token_limit} (enforcing minimum: {self.config.MAX_RETRY_TOKENS})")
else:
print(f" 🔄 TRUNCATION RETRY: Doubling tokens {old_limit} → {new_token_limit} (above minimum: {self.config.MAX_RETRY_TOKENS})")
else:
print(f" ⚠️ TRUNCATION DETECTED: Token adjustment not needed - already at maximum {self.config.MAX_OUTPUT_TOKENS}")
else:
print(f" ⚠️ TRUNCATION DETECTED: Max retries ({max_retries}) reached - accepting truncated response")
elif finish_reason == "length" and not (retry_truncated_enabled or self.config.RETRY_TRUNCATED):
print(f" ⏭️ TRUNCATION DETECTED: Auto-retry is DISABLED - accepting truncated response")
elif finish_reason == "length":
print(f" ⚠️ TRUNCATION DETECTED: Unexpected condition - check logic")
if not retry_needed:
# Force re-read the environment variable to ensure we have current setting
duplicate_enabled = os.getenv("RETRY_DUPLICATE_BODIES", "0") == "1"
if duplicate_enabled and duplicate_retry_count < max_duplicate_retries:
idx = c.get('__index', 0)
prog = c.get('__progress', {})
print(f" 🔍 Checking for duplicate content...")
# Get actual chapter number for duplicate detection
actual_num = c.get('actual_chapter_num', c.get('num', idx + 1))
is_duplicate, similarity = self.check_duplicate_content(result, idx, prog, self.out_dir, actual_num)
if is_duplicate:
retry_needed = True
is_duplicate_retry = True
retry_reason = f"duplicate content (similarity: {similarity}%)"
duplicate_retry_count += 1
# Check if temperature change is disabled
disable_temp_change = ai_config.get('disable_temperature_change', False) if isinstance(ai_config, dict) else False
if duplicate_retry_count >= 3 and not history_purged:
print(f" 🧹 Clearing history after 3 attempts...")
if 'history_manager' in c:
c['history_manager'].save_history([])
history_purged = True
if not disable_temp_change:
self.config.TEMP = original_temp
else:
print(f" 🌡️ Temperature change disabled - keeping current temp: {self.config.TEMP}")
elif duplicate_retry_count == 1:
if disable_temp_change:
print(f" 🔄 First duplicate retry - temperature change disabled")
else:
print(f" 🔄 First duplicate retry - same temperature")
elif history_purged:
if not disable_temp_change:
attempts_since_purge = duplicate_retry_count - 3
self.config.TEMP = min(original_temp + (0.1 * attempts_since_purge), 1.0)
print(f" 🌡️ Post-purge temp: {self.config.TEMP}")
else:
print(f" 🌡️ Temperature change disabled - keeping temp: {self.config.TEMP}")
else:
if not disable_temp_change:
self.config.TEMP = min(original_temp + (0.1 * (duplicate_retry_count - 1)), 1.0)
print(f" 🌡️ Gradual temp increase: {self.config.TEMP}")
else:
print(f" 🌡️ Temperature change disabled - keeping temp: {self.config.TEMP}")
if duplicate_retry_count == 1:
user_prompt = f"[RETRY] Chapter {c['num']}: Ensure unique translation.\n{chunk_html}"
elif duplicate_retry_count <= 3:
user_prompt = f"[ATTEMPT {duplicate_retry_count}] Translate uniquely:\n{chunk_html}"
else:
user_prompt = f"Chapter {c['num']}:\n{chunk_html}"
msgs[-1] = {"role": "user", "content": user_prompt}
elif not duplicate_enabled:
print(f" ⏭️ Duplicate detection is DISABLED - skipping check")
if retry_needed:
if is_duplicate_retry:
print(f" 🔄 Duplicate retry {duplicate_retry_count}/{max_duplicate_retries}")
else:
print(f" 🔄 Retry {retry_count}/{max_retries}: {retry_reason}")
time.sleep(2)
continue
break
except UnifiedClientError as e:
error_msg = str(e)
if "stopped by user" in error_msg:
print("❌ Translation stopped by user during API call")
return None, None
if "took" in error_msg and "timeout:" in error_msg:
if timeout_retry_count < max_timeout_retries:
timeout_retry_count += 1
print(f" ⏱️ Chunk took too long, retry {timeout_retry_count}/{max_timeout_retries}")
print(f" 🔄 Retrying")
time.sleep(2)
continue
else:
print(f" ❌ Max timeout retries reached")
raise UnifiedClientError("Translation failed after timeout retries")
elif "timed out" in error_msg and "timeout:" not in error_msg:
print(f"⚠️ {error_msg}, retrying...")
time.sleep(5)
continue
elif getattr(e, "error_type", None) == "rate_limit" or getattr(e, "http_status", None) == 429:
# Rate limit errors - clean handling without traceback
print("⚠️ Rate limited, sleeping 60s…")
for i in range(60):
if self.check_stop():
print("❌ Translation stopped during rate limit wait")
return None, None
time.sleep(1)
continue
else:
# For unexpected errors, show the error message but suppress traceback in most cases
if getattr(e, "error_type", None) in ["api_error", "validation", "prohibited_content"]:
print(f"❌ API Error: {error_msg}")
raise UnifiedClientError(f"API Error: {error_msg}")
else:
raise
except Exception as e:
print(f"❌ Unexpected error during API call: {e}")
raise
self.config.MAX_OUTPUT_TOKENS = original_max_tokens
self.config.TEMP = original_temp
if retry_count > 0 or duplicate_retry_count > 0 or timeout_retry_count > 0:
if duplicate_retry_count > 0:
print(f" 🔄 Restored original temperature: {self.config.TEMP} (after {duplicate_retry_count} duplicate retries)")
elif timeout_retry_count > 0:
print(f" 🔄 Restored original settings after {timeout_retry_count} timeout retries")
elif retry_count > 0:
print(f" 🔄 Restored original settings after {retry_count} retries")
if duplicate_retry_count >= max_duplicate_retries:
print(f" ⚠️ WARNING: Duplicate content issue persists after {max_duplicate_retries} attempts")
return result, finish_reason
def get_token_budget_str(self):
"""Get token budget as string"""
_tok_env = os.getenv("MAX_INPUT_TOKENS", "1000000").strip()
max_tokens_limit, budget_str = parse_token_limit(_tok_env)
return budget_str
# =====================================================
# BATCH TRANSLATION PROCESSOR
# =====================================================
class BatchTranslationProcessor:
"""Handles batch/parallel translation processing"""
def __init__(self, config, client, base_msg, out_dir, progress_lock,
save_progress_fn, update_progress_fn, check_stop_fn,
image_translator=None, is_text_file=False):
self.config = config
self.client = client
self.base_msg = base_msg
self.out_dir = out_dir
self.progress_lock = progress_lock
self.save_progress_fn = save_progress_fn
self.update_progress_fn = update_progress_fn
self.check_stop_fn = check_stop_fn
self.image_translator = image_translator
self.chapters_completed = 0
self.chunks_completed = 0
self.is_text_file = is_text_file
# Optionally log multi-key status
if hasattr(self.client, 'use_multi_keys') and self.client.use_multi_keys:
stats = self.client.get_stats()
print(f"🔑 Batch processor using multi-key mode: {stats.get('total_keys', 0)} keys")
def process_single_chapter(self, chapter_data):
"""Process a single chapter (runs in thread)"""
# APPLY INTERRUPTIBLE THREADING DELAY FIRST
thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
if thread_delay > 0:
# Check if we need to wait (same logic as unified_api_client)
if hasattr(self.client, '_thread_submission_lock') and hasattr(self.client, '_last_thread_submission_time'):
with self.client._thread_submission_lock:
current_time = time.time()
time_since_last = current_time - self.client._last_thread_submission_time
if time_since_last < thread_delay:
sleep_time = thread_delay - time_since_last
thread_name = threading.current_thread().name
# PRINT BEFORE THE DELAY STARTS
idx, chapter = chapter_data # Extract chapter info for better logging
print(f"🧵 [{thread_name}] Applying thread delay: {sleep_time:.1f}s for Chapter {idx+1}")
# Interruptible sleep - check stop flag every 0.1 seconds
elapsed = 0
check_interval = 0.1
while elapsed < sleep_time:
if self.check_stop_fn():
print(f"🛑 Threading delay interrupted by stop flag")
raise Exception("Translation stopped by user during threading delay")
sleep_chunk = min(check_interval, sleep_time - elapsed)
time.sleep(sleep_chunk)
elapsed += sleep_chunk
self.client._last_thread_submission_time = time.time()
if not hasattr(self.client, '_thread_submission_count'):
self.client._thread_submission_count = 0
self.client._thread_submission_count += 1
idx, chapter = chapter_data
chap_num = chapter["num"]
# Use the pre-calculated actual_chapter_num from the main loop
actual_num = chapter.get('actual_chapter_num')
# Fallback if not set (common in batch mode where first pass might be skipped)
if actual_num is None:
# Try to extract it using the same logic as non-batch mode
raw_num = FileUtilities.extract_actual_chapter_number(chapter, patterns=None, config=self.config)
# Apply offset if configured
offset = self.config.CHAPTER_NUMBER_OFFSET if hasattr(self.config, 'CHAPTER_NUMBER_OFFSET') else 0
raw_num += offset
# Check if zero detection is disabled
if hasattr(self.config, 'DISABLE_ZERO_DETECTION') and self.config.DISABLE_ZERO_DETECTION:
actual_num = raw_num
elif hasattr(self.config, '_uses_zero_based') and self.config._uses_zero_based:
# This is a 0-based novel, adjust the number
actual_num = raw_num + 1
else:
# Default to raw number (1-based or unknown)
actual_num = raw_num
print(f" 📖 Extracted actual chapter number: {actual_num} (from raw: {raw_num})")
try:
# Check if this is from a text file
ai_features = None
is_text_source = self.is_text_file or chapter.get('filename', '').endswith('.txt') or chapter.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
print(f"🔄 Starting #{idx+1} (Internal: {terminology} {chap_num}, Actual: {terminology} {actual_num}) (thread: {threading.current_thread().name}) [File: {chapter.get('original_basename', f'{terminology}_{chap_num}')}]")
content_hash = chapter.get("content_hash") or ContentProcessor.get_content_hash(chapter["body"])
with self.progress_lock:
self.update_progress_fn(idx, actual_num, content_hash, None, status="in_progress")
self.save_progress_fn()
chapter_body = chapter["body"]
if chapter.get('has_images') and self.image_translator and self.config.ENABLE_IMAGE_TRANSLATION:
print(f"🖼️ Processing images for Chapter {actual_num}...")
self.image_translator.set_current_chapter(actual_num)
chapter_body, image_translations = process_chapter_images(
chapter_body,
actual_num,
self.image_translator,
self.check_stop_fn
)
if image_translations:
# Create a copy of the processed body
from bs4 import BeautifulSoup
c = chapter
soup_for_text = BeautifulSoup(c["body"], 'html.parser')
# Remove all translated content
for trans_div in soup_for_text.find_all('div', class_='translated-text-only'):
trans_div.decompose()
# Use this cleaned version for text translation
text_to_translate = str(soup_for_text)
final_body_with_images = c["body"]
else:
text_to_translate = c["body"]
image_translations = {}
print(f"✅ Processed {len(image_translations)} images for Chapter {actual_num}")
chapter_msgs = self.base_msg + [{"role": "user", "content": chapter_body}]
# Generate filename before API call
fname = FileUtilities.create_chapter_filename(chapter, actual_num)
self.client.set_output_filename(fname)
if hasattr(self.client, '_current_output_file'):
self.client._current_output_file = fname
print(f"📤 Sending Chapter {actual_num} to API...")
result, finish_reason = send_with_interrupt(
chapter_msgs, self.client, self.config.TEMP,
self.config.MAX_OUTPUT_TOKENS, self.check_stop_fn
)
print(f"📥 Received Chapter {actual_num} response, finish_reason: {finish_reason}")
# Enhanced mode workflow (same as non-batch):
# 1. Original HTML -> html2text -> Markdown/plain text (during extraction)
# 2. Markdown sent to translation API (better for translation quality)
# 3. Translated markdown -> HTML conversion (here)
if result and chapter.get("enhanced_extraction", False):
print(f"🔄 Converting translated markdown back to HTML...")
result = convert_enhanced_text_to_html(result, chapter)
if finish_reason in ["length", "max_tokens"]:
print(f"⚠️ Chapter {actual_num} response was TRUNCATED!")
if self.config.REMOVE_AI_ARTIFACTS:
result = ContentProcessor.clean_ai_artifacts(result, True)
result = ContentProcessor.clean_memory_artifacts(result)
cleaned = re.sub(r"^```(?:html)?\s*\n?", "", result, count=1, flags=re.MULTILINE)
cleaned = re.sub(r"\n?```\s*$", "", cleaned, count=1, flags=re.MULTILINE)
cleaned = ContentProcessor.clean_ai_artifacts(cleaned, remove_artifacts=self.config.REMOVE_AI_ARTIFACTS)
fname = FileUtilities.create_chapter_filename(chapter, actual_num)
if self.is_text_file:
# For text files, save as plain text
fname_txt = fname.replace('.html', '.txt') if fname.endswith('.html') else fname
# Extract text from HTML
from bs4 import BeautifulSoup
soup = BeautifulSoup(cleaned, 'html.parser')
text_content = soup.get_text(strip=True)
# Merge image translations back with text translation
if 'final_body_with_images' in locals() and image_translations:
# Parse both versions
soup_with_images = BeautifulSoup(final_body_with_images, 'html.parser')
soup_with_text = BeautifulSoup(cleaned, 'html.parser')
# Get the translated text content (without images)
body_content = soup_with_text.body
# Add image translations to the translated content
for trans_div in soup_with_images.find_all('div', class_='translated-text-only'):
body_content.insert(0, trans_div)
final_html = str(soup_with_text)
cleaned = final_html
with open(os.path.join(self.out_dir, fname), 'w', encoding='utf-8') as f:
f.write(cleaned)
# Update with .txt filename
with self.progress_lock:
self.update_progress_fn(idx, actual_num, content_hash, fname_txt, status="completed", ai_features=ai_features)
self.save_progress_fn()
else:
# Original code for EPUB files
with open(os.path.join(self.out_dir, fname), 'w', encoding='utf-8') as f:
f.write(cleaned)
print(f"💾 Saved Chapter {actual_num}: {fname} ({len(cleaned)} chars)")
# Initialize ai_features at the beginning to ensure it's always defined
if ai_features is None:
ai_features = None
# Extract and save AI features for future duplicate detection
if (self.config.RETRY_DUPLICATE_BODIES and
hasattr(self.config, 'DUPLICATE_DETECTION_MODE') and
self.config.DUPLICATE_DETECTION_MODE in ['ai-hunter', 'cascading']):
try:
# Extract features from the translated content
cleaned_text = re.sub(r'<[^>]+>', '', cleaned).strip()
# Note: self.translator doesn't exist, so we can't extract features here
# The features will need to be extracted during regular processing
print(f" ⚠️ AI features extraction not available in batch mode")
except Exception as e:
print(f" ⚠️ Failed to extract AI features: {e}")
with self.progress_lock:
# Check for QA failures with comprehensive detection
if is_qa_failed_response(cleaned):
chapter_status = "qa_failed"
failure_reason = get_failure_reason(cleaned)
print(f"⚠️ Batch: Chapter {actual_num} marked as qa_failed: {failure_reason}")
# Update progress to qa_failed status
self.update_progress_fn(idx, actual_num, content_hash, fname, status=chapter_status, ai_features=ai_features)
self.save_progress_fn()
# DO NOT increment chapters_completed for qa_failed
# Return False to indicate failure
return False, actual_num
else:
chapter_status = "completed"
# Update progress to completed status
self.update_progress_fn(idx, actual_num, content_hash, fname, status=chapter_status, ai_features=ai_features)
self.save_progress_fn()
# Only increment chapters_completed for successful chapters
self.chapters_completed += 1
self.chunks_completed += 1
print(f"✅ Chapter {actual_num} completed successfully")
return True, actual_num
except Exception as e:
print(f"❌ Chapter {actual_num} failed: {e}")
with self.progress_lock:
self.update_progress_fn(idx, actual_num, content_hash, None, status="failed")
self.save_progress_fn()
return False, actual_num
# =====================================================
# GLOSSARY MANAGER - TRUE CSV FORMAT WITH FUZZY MATCHING
# =====================================================
class GlossaryManager:
"""Unified glossary management with true CSV format, fuzzy matching, and parallel processing"""
# Class-level shared lock for API submission timing
_api_submission_lock = threading.Lock()
_last_api_submission_time = 0
def __init__(self):
self.pattern_manager = PatternManager()
self._results_lock = threading.Lock() # Thread lock for collecting results
self._file_write_lock = threading.Lock() # Thread lock for file operations
def _atomic_write_file(self, filepath, content, encoding='utf-8'):
"""Atomically write to a file to prevent corruption from concurrent writes"""
# Create temp file in same directory to ensure same filesystem
dir_path = os.path.dirname(filepath)
with self._file_write_lock:
try:
# Write to temporary file first
with tempfile.NamedTemporaryFile(mode='w', encoding=encoding,
dir=dir_path, delete=False) as tmp_file:
tmp_file.write(content)
tmp_path = tmp_file.name
# Atomic rename (on same filesystem)
if os.name == 'nt': # Windows
# Windows doesn't support atomic rename if target exists
if os.path.exists(filepath):
os.remove(filepath)
os.rename(tmp_path, filepath)
else: # Unix/Linux/Mac
os.rename(tmp_path, filepath)
return True
except Exception as e:
print(f"⚠️ Atomic write failed: {e}")
# Cleanup temp file if it exists
if 'tmp_path' in locals() and os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except:
pass
# Fallback to direct write with lock
try:
with open(filepath, 'w', encoding=encoding) as f:
f.write(content)
return True
except Exception as e2:
print(f"⚠️ Fallback write also failed: {e2}")
return False
def save_glossary(self, output_dir, chapters, instructions, language="korean"):
"""Targeted glossary generator with true CSV format output and parallel processing"""
print("📑 Targeted Glossary Generator v6.0 (CSV Format + Parallel)")
# Check stop flag at start
# Ensure output directory exists
try:
os.makedirs(output_dir, exist_ok=True)
except Exception as _e:
print(f"⚠️ Could not ensure output directory exists: {output_dir} ({_e})")
if is_stop_requested():
print("📑 ❌ Glossary generation stopped by user")
return {}
# Check if glossary already exists; if so, we'll MERGE it later (do not return early)
glossary_path = os.path.join(output_dir, "glossary.csv")
existing_glossary_content = None
if os.path.exists(glossary_path):
print(f"📑 Existing glossary detected (will merge): {glossary_path}")
try:
with open(glossary_path, 'r', encoding='utf-8') as f:
existing_glossary_content = f.read()
except Exception as e:
print(f"⚠️ Could not read existing glossary: {e}")
# Rest of the method continues as before...
print("📑 Extracting names and terms with configurable options")
# Check stop flag before processing
if is_stop_requested():
print("📑 ❌ Glossary generation stopped by user")
return {}
# Check for manual glossary first (CSV only)
manual_glossary_path = os.getenv("MANUAL_GLOSSARY")
existing_glossary = None
if manual_glossary_path and os.path.exists(manual_glossary_path):
print(f"📑 Manual glossary detected: {os.path.basename(manual_glossary_path)}")
try:
with open(manual_glossary_path, 'r', encoding='utf-8') as f:
content = f.read()
# Treat as CSV text and stage it for merge; also copy to output for visibility
target_path = os.path.join(output_dir, "glossary.csv")
with open(target_path, 'w', encoding='utf-8') as f:
f.write(content)
print(f"📑 ✅ Manual CSV glossary copied to: {target_path}")
existing_glossary = content
except Exception as e:
print(f"⚠️ Could not copy manual glossary: {e}")
print(f"📑 Proceeding with automatic generation...")
# Check for existing glossary from manual extraction
glossary_folder_path = os.path.join(output_dir, "Glossary")
# existing_glossary may already be set by MANUAL_GLOSSARY above
if os.path.exists(glossary_folder_path):
for file in os.listdir(glossary_folder_path):
if file.endswith("_glossary.json"):
existing_path = os.path.join(glossary_folder_path, file)
try:
with open(existing_path, 'r', encoding='utf-8') as f:
existing_content = f.read()
existing_glossary = existing_content
print(f"📑 Found existing glossary from manual extraction: {file}")
break
except Exception as e:
print(f"⚠️ Could not load existing glossary: {e}")
# Get configuration from environment variables
min_frequency = int(os.getenv("GLOSSARY_MIN_FREQUENCY", "2"))
max_names = int(os.getenv("GLOSSARY_MAX_NAMES", "50"))
max_titles = int(os.getenv("GLOSSARY_MAX_TITLES", "30"))
batch_size = int(os.getenv("GLOSSARY_BATCH_SIZE", "50"))
strip_honorifics = os.getenv("GLOSSARY_STRIP_HONORIFICS", "1") == "1"
fuzzy_threshold = float(os.getenv("GLOSSARY_FUZZY_THRESHOLD", "0.90"))
max_text_size = int(os.getenv("GLOSSARY_MAX_TEXT_SIZE", "50000"))
print(f"📑 Settings: Min frequency: {min_frequency}, Max names: {max_names}, Max titles: {max_titles}")
print(f"📑 Strip honorifics: {'✅ Yes' if strip_honorifics else '❌ No'}")
print(f"📑 Fuzzy matching threshold: {fuzzy_threshold}")
# Get custom prompt from environment
custom_prompt = os.getenv("AUTO_GLOSSARY_PROMPT", "").strip()
def clean_html(html_text):
"""Remove HTML tags to get clean text"""
soup = BeautifulSoup(html_text, 'html.parser')
return soup.get_text()
# Check stop before processing chapters
if is_stop_requested():
print("📑 ❌ Glossary generation stopped by user")
return {}
# Get chapter split threshold and filter mode
chapter_split_threshold = int(os.getenv("GLOSSARY_CHAPTER_SPLIT_THRESHOLD", "100000"))
filter_mode = os.getenv("GLOSSARY_FILTER_MODE", "all") # all, only_with_honorifics, only_without_honorifics
# Check if parallel extraction is enabled for automatic glossary
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
batch_translation = os.getenv("BATCH_TRANSLATION", "0") == "1"
api_batch_size = int(os.getenv("BATCH_SIZE", "5"))
# Log the settings
print(f"📑 Filter mode: {filter_mode}")
if extraction_workers > 1:
print(f"📑 Parallel extraction enabled: {extraction_workers} workers")
if batch_translation:
print(f"📑 Batch API calls enabled: {api_batch_size} chunks per batch")
all_text = ' '.join(clean_html(chapter["body"]) for chapter in chapters)
print(f"📑 Processing {len(all_text):,} characters of text")
# Apply smart filtering FIRST to check actual size needed
use_smart_filter = os.getenv("GLOSSARY_USE_SMART_FILTER", "1") == "1"
effective_text_size = len(all_text)
filtered_text_cache = None
if use_smart_filter and custom_prompt: # Only apply for AI extraction
print(f"📑 Smart filtering enabled - checking effective text size after filtering...")
# Perform filtering ONCE and reuse for chunking
filtered_sample, _ = self._filter_text_for_glossary(all_text, min_frequency)
filtered_text_cache = filtered_sample
effective_text_size = len(filtered_sample)
print(f"📑 Effective text size after filtering: {effective_text_size:,} chars (from {len(all_text):,})")
# Check if we need to split into chunks based on EFFECTIVE size after filtering
if chapter_split_threshold > 0 and effective_text_size > chapter_split_threshold:
print(f"📑 Effective text exceeds {chapter_split_threshold:,} chars, will process in chunks...")
# If using smart filter, we need to split the FILTERED text, not raw text
if use_smart_filter and custom_prompt:
# Split the filtered text into chunks (reuse cached filtered text)
filtered_text = filtered_text_cache if filtered_text_cache is not None else self._filter_text_for_glossary(all_text, min_frequency)[0]
chunks_to_process = []
# Split filtered text into chunks of appropriate size
chunk_size = chapter_split_threshold
for i in range(0, len(filtered_text), chunk_size):
chunk_text = filtered_text[i:i + chunk_size]
chunks_to_process.append((len(chunks_to_process) + 1, chunk_text))
print(f"📑 Split filtered text into {len(chunks_to_process)} chunks")
all_glossary_entries = []
else:
# Original logic for unfiltered text
all_glossary_entries = []
chunk_size = 0
chunk_chapters = []
chunks_to_process = []
for idx, chapter in enumerate(chapters):
if is_stop_requested():
print("📑 ❌ Glossary generation stopped by user")
return all_glossary_entries
chapter_text = clean_html(chapter["body"])
chunk_size += len(chapter_text)
chunk_chapters.append(chapter)
# Process chunk when it reaches threshold or last chapter
if chunk_size >= chapter_split_threshold or idx == len(chapters) - 1:
chunk_text = ' '.join(clean_html(ch["body"]) for ch in chunk_chapters)
chunks_to_process.append((len(chunks_to_process) + 1, chunk_text))
# Reset for next chunk
chunk_size = 0
chunk_chapters = []
print(f"📑 Split into {len(chunks_to_process)} chunks for processing")
# Batch toggle decides concurrency: ON => parallel API calls; OFF => strict sequential
if batch_translation and custom_prompt and len(chunks_to_process) > 1:
print(f"📑 Processing chunks in batch mode with {api_batch_size} chunks per batch...")
# Set fast mode for batch processing
os.environ["GLOSSARY_SKIP_ALL_VALIDATION"] = "1"
# Use batch API calls for AI extraction
all_csv_lines = self._process_chunks_batch_api(
chunks_to_process, custom_prompt, language,
min_frequency, max_names, max_titles,
output_dir, strip_honorifics, fuzzy_threshold,
filter_mode, api_batch_size, extraction_workers
)
# Reset validation mode
os.environ["GLOSSARY_SKIP_ALL_VALIDATION"] = "0"
print(f"📑 All chunks completed. Aggregated raw lines: {len(all_csv_lines)}")
# Process all collected entries at once (even if empty)
# Add header so downstream steps can work uniformly
all_csv_lines.insert(0, "type,raw_name,translated_name")
# Merge with any on-disk glossary first (to avoid overwriting user edits)
on_disk_path = os.path.join(output_dir, "glossary.csv")
if os.path.exists(on_disk_path):
try:
with open(on_disk_path, 'r', encoding='utf-8') as f:
on_disk_content = f.read()
all_csv_lines = self._merge_csv_entries(all_csv_lines, on_disk_content, strip_honorifics, language)
print("📑 Merged with existing on-disk glossary")
except Exception as e:
print(f"⚠️ Failed to merge with existing on-disk glossary: {e}")
# Apply filter mode if needed
if filter_mode == "only_with_honorifics":
filtered = [all_csv_lines[0]] # Keep header
for line in all_csv_lines[1:]:
parts = line.split(',', 2)
if len(parts) >= 3 and parts[0] == "character":
filtered.append(line)
all_csv_lines = filtered
print(f"📑 Filter applied: {len(all_csv_lines)-1} character entries with honorifics kept")
# Apply fuzzy deduplication (deferred until after all chunks)
try:
print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
all_csv_lines = self._deduplicate_glossary_with_fuzzy(all_csv_lines, fuzzy_threshold)
except Exception as e:
print(f"⚠️ Deduplication error: {e} — continuing without dedup")
# Sort by type and name
print(f"📑 Sorting glossary by type and name...")
header = all_csv_lines[0]
entries = all_csv_lines[1:]
if entries:
entries.sort(key=lambda x: (0 if x.startswith('character,') else 1, x.split(',')[1].lower()))
all_csv_lines = [header] + entries
# Save
# Check format preference
use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
if not use_legacy_format:
# Convert to token-efficient format
all_csv_lines = self._convert_to_token_efficient_format(all_csv_lines)
# Final sanitize to prevent stray headers
all_csv_lines = self._sanitize_final_glossary_lines(all_csv_lines, use_legacy_format)
# Save
csv_content = '\n'.join(all_csv_lines)
glossary_path = os.path.join(output_dir, "glossary.csv")
self._atomic_write_file(glossary_path, csv_content)
# Verify file exists; fallback direct write if needed
if not os.path.exists(glossary_path):
try:
with open(glossary_path, 'w', encoding='utf-8') as f:
f.write(csv_content)
print("📑 Fallback write succeeded for glossary.csv")
except Exception as e:
print(f"❌ Failed to write glossary.csv: {e}")
print(f"\n📑 ✅ GLOSSARY SAVED!")
print(f"📑 ✅ AI GLOSSARY SAVED!")
c_count, t_count, total = self._count_glossary_entries(all_csv_lines, use_legacy_format)
print(f"📑 Character entries: {c_count}")
print(f"📑 Term entries: {t_count}")
print(f"📑 Total entries: {total}")
return self._parse_csv_to_dict(csv_content)
else:
# Strict sequential processing (one API call at a time)
_prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
_prev_filtered = os.getenv("_CHUNK_ALREADY_FILTERED")
_prev_force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER")
os.environ["GLOSSARY_DEFER_SAVE"] = "1"
# Tell the extractor each chunk is already filtered to avoid re-running smart filter per chunk
os.environ["_CHUNK_ALREADY_FILTERED"] = "1"
os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = "1"
try:
for chunk_idx, chunk_text in chunks_to_process:
if is_stop_requested():
break
print(f"📑 Processing chunk {chunk_idx}/{len(chunks_to_process)} ({len(chunk_text):,} chars)...")
if custom_prompt:
chunk_glossary = self._extract_with_custom_prompt(
custom_prompt, chunk_text, language,
min_frequency, max_names, max_titles,
None, output_dir, # Don't pass existing glossary to chunks
strip_honorifics, fuzzy_threshold, filter_mode
)
else:
chunk_glossary = self._extract_with_patterns(
chunk_text, language, min_frequency,
max_names, max_titles, batch_size,
None, output_dir, # Don't pass existing glossary to chunks
strip_honorifics, fuzzy_threshold, filter_mode
)
# Normalize to CSV lines and aggregate
chunk_lines = []
if isinstance(chunk_glossary, list):
for line in chunk_glossary:
if line and not line.startswith('type,'):
all_glossary_entries.append(line)
chunk_lines.append(line)
else:
for raw_name, translated_name in chunk_glossary.items():
entry_type = "character" if self._has_honorific(raw_name) else "term"
line = f"{entry_type},{raw_name},{translated_name}"
all_glossary_entries.append(line)
chunk_lines.append(line)
# Incremental update
try:
self._incremental_update_glossary(output_dir, chunk_lines, strip_honorifics, language, filter_mode)
print(f"📑 Incremental write: +{len(chunk_lines)} entries")
except Exception as e2:
print(f"⚠️ Incremental write failed: {e2}")
finally:
if _prev_defer is None:
if "GLOSSARY_DEFER_SAVE" in os.environ:
del os.environ["GLOSSARY_DEFER_SAVE"]
else:
os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
if _prev_filtered is None:
os.environ.pop("_CHUNK_ALREADY_FILTERED", None)
else:
os.environ["_CHUNK_ALREADY_FILTERED"] = _prev_filtered
if _prev_force_disable is None:
os.environ.pop("GLOSSARY_FORCE_DISABLE_SMART_FILTER", None)
else:
os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = _prev_force_disable
# Build CSV from aggregated entries
csv_lines = ["type,raw_name,translated_name"] + all_glossary_entries
# Merge with any provided existing glossary AND on-disk glossary to avoid overwriting
on_disk_path = os.path.join(output_dir, "glossary.csv")
merge_sources = []
if existing_glossary:
merge_sources.append(existing_glossary)
if os.path.exists(on_disk_path):
try:
with open(on_disk_path, 'r', encoding='utf-8') as f:
merge_sources.append(f.read())
print("📑 Found existing on-disk glossary to merge")
except Exception as e:
print(f"⚠️ Failed to read on-disk glossary for merging: {e}")
# Also merge the main on-disk glossary if it was present at start
if existing_glossary_content:
csv_lines = self._merge_csv_entries(csv_lines, existing_glossary_content, strip_honorifics, language)
for src in merge_sources:
csv_lines = self._merge_csv_entries(csv_lines, src, strip_honorifics, language)
# Apply filter mode to final results
csv_lines = self._filter_csv_by_mode(csv_lines, filter_mode)
# Apply fuzzy deduplication (deferred until after all chunks)
print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
original_count = len(csv_lines) - 1
csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
deduped_count = len(csv_lines) - 1
if original_count > deduped_count:
print(f"📑 Removed {original_count - deduped_count} duplicate entries")
# Sort by type and name
print(f"📑 Sorting glossary by type and name...")
header = csv_lines[0]
entries = csv_lines[1:]
entries.sort(key=lambda x: (0 if x.startswith('character,') else 1, x.split(',')[1].lower() if ',' in x else x.lower()))
csv_lines = [header] + entries
# Token-efficient format if enabled
use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
if not use_legacy_format:
csv_lines = self._convert_to_token_efficient_format(csv_lines)
# Final sanitize to prevent stray headers and section titles at end
csv_lines = self._sanitize_final_glossary_lines(csv_lines, use_legacy_format)
try:
# Save
csv_content = '\n'.join(csv_lines)
glossary_path = os.path.join(output_dir, "glossary.csv")
self._atomic_write_file(glossary_path, csv_content)
# Verify file exists; fallback direct write if needed
if not os.path.exists(glossary_path):
try:
with open(glossary_path, 'w', encoding='utf-8') as f:
f.write(csv_content)
print("📑 Fallback write succeeded for glossary.csv")
except Exception as e:
print(f"❌ Failed to write glossary.csv: {e}")
finally:
print(f"\n📑 ✅ CHUNKED GLOSSARY SAVED!")
print(f"📑 ✅ AI GLOSSARY SAVED!")
print(f"📑 File: {glossary_path}")
c_count, t_count, total = self._count_glossary_entries(csv_lines, use_legacy_format)
print(f"📑 Character entries: {c_count}")
print(f"📑 Term entries: {t_count}")
print(f"📑 Total entries: {total}")
return self._parse_csv_to_dict(csv_content)
# Original single-text processing
if custom_prompt:
return self._extract_with_custom_prompt(custom_prompt, all_text, language,
min_frequency, max_names, max_titles,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
else:
return self._extract_with_patterns(all_text, language, min_frequency,
max_names, max_titles, batch_size,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
total_time = time.time() - total_start_time
print(f"\n📑 ========== GLOSSARY GENERATION COMPLETE ==========")
print(f"📑 Total time: {total_time:.1f}s")
print(f"📑 Performance breakdown:")
print(f"📑 - Extraction: {getattr(self, '_extraction_time', 0):.1f}s")
print(f"📑 - API calls: {getattr(self, '_api_time', 0):.1f}s")
print(f"📑 - Frequency checking: {getattr(self, '_freq_check_time', 0):.1f}s")
print(f"📑 - Deduplication: {getattr(self, '_dedup_time', 0):.1f}s")
print(f"📑 - File I/O: {getattr(self, '_io_time', 0):.1f}s")
print(f"📑 ================================================")
return result # This is the existing return statement
def _convert_to_token_efficient_format(self, csv_lines):
"""Convert CSV lines to token-efficient format with sections and asterisks"""
if len(csv_lines) <= 1:
return csv_lines
header = csv_lines[0]
entries = csv_lines[1:]
# Group by type (only from valid CSV lines)
import re as _re
grouped = {}
for line in entries:
if not line.strip():
continue
# Only accept proper CSV rows: at least 3 fields and a sane type token
parts_full = [p.strip() for p in line.split(',')]
if len(parts_full) < 3:
continue
entry_type = parts_full[0].lower()
if not _re.match(r'^[a-z_]+$', entry_type):
continue
if entry_type not in grouped:
grouped[entry_type] = []
grouped[entry_type].append(line)
# Rebuild with token-efficient format
result = []
result.append("Glossary: Characters, Terms, and Important Elements\n")
# Process in order: character first, then term, then others
type_order = ['character', 'term'] + [t for t in grouped.keys() if t not in ['character', 'term']]
for entry_type in type_order:
if entry_type not in grouped:
continue
entries = grouped[entry_type]
# Add section header
section_name = entry_type.upper() + 'S' if not entry_type.upper().endswith('S') else entry_type.upper()
result.append(f"=== {section_name} ===")
# Add entries in new format
for line in entries:
parts = [p.strip() for p in line.split(',')]
if len(parts) >= 3:
raw_name = parts[1]
translated_name = parts[2]
# Format: * TranslatedName (RawName)
entry_line = f"* {translated_name} ({raw_name})"
# Add gender if present and not Unknown
if len(parts) > 3 and parts[3] and parts[3] != 'Unknown':
entry_line += f" [{parts[3]}]"
# Add any additional fields as description
if len(parts) > 4:
description = ', '.join(parts[4:])
if description.strip():
entry_line += f": {description}"
result.append(entry_line)
result.append("") # Blank line between sections
return result
def _count_glossary_entries(self, lines, use_legacy_format=False):
"""Return (char_count, term_count, total_count) for either format."""
if not lines:
return 0, 0, 0
if use_legacy_format:
data = lines[1:] if lines and lines[0].lower().startswith('type,raw_name') else lines
char_count = sum(1 for ln in data if ln.startswith('character,'))
term_count = sum(1 for ln in data if ln.startswith('term,'))
total = sum(1 for ln in data if ln and ',' in ln)
return char_count, term_count, total
# token-efficient
current = None
char_count = term_count = total = 0
for ln in lines:
s = ln.strip()
if s.startswith('=== ') and 'CHARACTER' in s.upper():
current = 'character'
continue
if s.startswith('=== ') and 'TERM' in s.upper():
current = 'term'
continue
if s.startswith('* '):
total += 1
if current == 'character':
char_count += 1
elif current == 'term':
term_count += 1
return char_count, term_count, total
def _sanitize_final_glossary_lines(self, lines, use_legacy_format=False):
"""Remove stray CSV headers and normalize header placement before saving.
- In legacy CSV mode, ensure exactly one header at the very top.
- In token-efficient mode, remove any CSV header lines entirely.
"""
header_norm = "type,raw_name,translated_name"
if not lines:
return lines
if use_legacy_format:
sanitized = []
header_seen = False
for ln in lines:
txt = ln.strip()
if txt.lower().startswith("type,raw_name"):
if not header_seen:
sanitized.append(header_norm)
header_seen = True
# skip duplicates
else:
sanitized.append(ln)
# ensure header at top
if sanitized and not sanitized[0].strip().lower().startswith("type,raw_name"):
sanitized.insert(0, header_norm)
return sanitized
else:
# remove any CSV header lines anywhere and duplicate top headers/sections
cleaned = []
glossary_header_seen = False
for i, ln in enumerate(lines):
txt = ln.strip()
low = txt.lower()
# Drop CSV headers
if low.startswith("type,raw_name"):
continue
# Keep only the first main glossary header
if low.startswith("glossary:"):
if glossary_header_seen:
continue
glossary_header_seen = True
cleaned.append(ln)
continue
# Remove bogus section like '=== GLOSSARY: ... ==='
if low.startswith("=== glossary:"):
continue
cleaned.append(ln)
return cleaned
def _process_chunks_batch_api(self, chunks_to_process, custom_prompt, language,
min_frequency, max_names, max_titles,
output_dir, strip_honorifics, fuzzy_threshold,
filter_mode, api_batch_size, extraction_workers):
"""Process chunks using batch API calls for AI extraction with thread delay"""
print(f"📑 Using batch API mode with {api_batch_size} chunks per batch")
# Ensure we defer saving and heavy merging when processing chunks
_prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
os.environ["GLOSSARY_DEFER_SAVE"] = "1"
# Get thread submission delay
thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
if thread_delay > 0:
print(f"📑 Thread submission delay: {thread_delay}s between parallel calls")
# CHANGE: Collect raw CSV lines instead of dictionary
all_csv_lines = [] # Collect all entries as CSV lines
total_chunks = len(chunks_to_process)
completed_chunks = 0
# Ensure per-chunk smart filtering is disabled globally during batch processing
_prev_filtered = os.getenv("_CHUNK_ALREADY_FILTERED")
_prev_force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER")
os.environ["_CHUNK_ALREADY_FILTERED"] = "1"
os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = "1"
# Process in API batches
for batch_start in range(0, len(chunks_to_process), api_batch_size):
if is_stop_requested():
break
batch_end = min(batch_start + api_batch_size, len(chunks_to_process))
batch_chunks = chunks_to_process[batch_start:batch_end]
print(f"📑 Processing API batch {batch_start//api_batch_size + 1}: chunks {batch_start+1}-{batch_end}")
# Use ThreadPoolExecutor for parallel API calls within batch
# Batch mode: issue multiple API calls in parallel within each batch (one worker per chunk)
with ThreadPoolExecutor(max_workers=len(batch_chunks)) as executor:
futures = {}
last_submission_time = 0
for chunk_idx, chunk_text in batch_chunks:
if is_stop_requested():
break
# Apply thread submission delay
if thread_delay > 0 and last_submission_time > 0:
time_since_last = time.time() - last_submission_time
if time_since_last < thread_delay:
sleep_time = thread_delay - time_since_last
print(f"🧵 Thread delay: {sleep_time:.1f}s for chunk {chunk_idx}")
time.sleep(sleep_time)
future = executor.submit(
self._extract_with_custom_prompt,
custom_prompt, chunk_text, language,
min_frequency, max_names, max_titles,
None, output_dir, strip_honorifics,
fuzzy_threshold, filter_mode
)
futures[future] = chunk_idx
last_submission_time = time.time()
# Collect results
for future in as_completed(futures):
if is_stop_requested():
break
try:
chunk_glossary = future.result()
print(f"📑 DEBUG: Chunk {futures[future]} returned type={type(chunk_glossary)}, len={len(chunk_glossary)}")
# Normalize to CSV lines (without header)
chunk_lines = []
if isinstance(chunk_glossary, dict):
for raw_name, translated_name in chunk_glossary.items():
entry_type = "character" if self._has_honorific(raw_name) else "term"
chunk_lines.append(f"{entry_type},{raw_name},{translated_name}")
elif isinstance(chunk_glossary, list):
for line in chunk_glossary:
if line and not line.startswith('type,'):
chunk_lines.append(line)
# Aggregate for end-of-run
all_csv_lines.extend(chunk_lines)
# Incremental update of glossary.csv in token-efficient format
try:
self._incremental_update_glossary(output_dir, chunk_lines, strip_honorifics, language, filter_mode)
print(f"📑 Incremental write: +{len(chunk_lines)} entries")
except Exception as e2:
print(f"⚠️ Incremental write failed: {e2}")
completed_chunks += 1
# Print progress for GUI
progress_percent = (completed_chunks / total_chunks) * 100
print(f"📑 Progress: {completed_chunks}/{total_chunks} chunks ({progress_percent:.0f}%)")
print(f"📑 Chunk {futures[future]} completed and aggregated")
except Exception as e:
print(f"⚠️ API call for chunk {futures[future]} failed: {e}")
completed_chunks += 1
progress_percent = (completed_chunks / total_chunks) * 100
print(f"📑 Progress: {completed_chunks}/{total_chunks} chunks ({progress_percent:.0f}%)")
# Add delay between API batches
if batch_end < len(chunks_to_process):
api_delay = float(os.getenv("SEND_INTERVAL_SECONDS", "2"))
print(f"⏱️ Waiting {api_delay}s before next API batch...")
time.sleep(api_delay)
# CHANGE: Return CSV lines instead of dictionary
# Restore per-chunk filter disabling envs
if _prev_filtered is None:
os.environ.pop("_CHUNK_ALREADY_FILTERED", None)
else:
os.environ["_CHUNK_ALREADY_FILTERED"] = _prev_filtered
if _prev_force_disable is None:
os.environ.pop("GLOSSARY_FORCE_DISABLE_SMART_FILTER", None)
else:
os.environ["GLOSSARY_FORCE_DISABLE_SMART_FILTER"] = _prev_force_disable
# Restore previous defer setting
if _prev_defer is None:
# Default back to not deferring if it wasn't set
if "GLOSSARY_DEFER_SAVE" in os.environ:
del os.environ["GLOSSARY_DEFER_SAVE"]
else:
os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
return all_csv_lines
def _incremental_update_glossary(self, output_dir, chunk_lines, strip_honorifics, language, filter_mode):
"""Incrementally update glossary.csv (token-efficient) using an on-disk CSV aggregator.
This keeps glossary.csv present and growing after each chunk while preserving
token-efficient format for the visible file.
"""
if not chunk_lines:
return
# Paths
agg_path = os.path.join(output_dir, "glossary.incremental.csv")
vis_path = os.path.join(output_dir, "glossary.csv")
# Ensure output dir
os.makedirs(output_dir, exist_ok=True)
# Compose CSV with header for merging
new_csv_lines = ["type,raw_name,translated_name"] + chunk_lines
# Load existing aggregator content, if any
existing_csv = None
if os.path.exists(agg_path):
try:
with open(agg_path, 'r', encoding='utf-8') as f:
existing_csv = f.read()
except Exception as e:
print(f"⚠️ Incremental: cannot read aggregator: {e}")
# Merge (exact merge, no fuzzy to keep this fast)
merged_csv_lines = self._merge_csv_entries(new_csv_lines, existing_csv or "", strip_honorifics, language)
# Optional filter mode
merged_csv_lines = self._filter_csv_by_mode(merged_csv_lines, filter_mode)
# Save aggregator (CSV)
self._atomic_write_file(agg_path, "\n".join(merged_csv_lines))
# Convert to token-efficient format for visible glossary.csv
token_lines = self._convert_to_token_efficient_format(merged_csv_lines)
token_lines = self._sanitize_final_glossary_lines(token_lines, use_legacy_format=False)
self._atomic_write_file(vis_path, "\n".join(token_lines))
if not os.path.exists(vis_path):
with open(vis_path, 'w', encoding='utf-8') as f:
f.write("\n".join(token_lines))
def _process_single_chunk(self, chunk_idx, chunk_text, custom_prompt, language,
min_frequency, max_names, max_titles, batch_size,
output_dir, strip_honorifics, fuzzy_threshold, filter_mode,
already_filtered=False):
"""Process a single chunk - wrapper for parallel execution"""
print(f"📑 Worker processing chunk {chunk_idx} ({len(chunk_text):,} chars)...")
if custom_prompt:
# Pass flag to indicate if text is already filtered
os.environ["_CHUNK_ALREADY_FILTERED"] = "1" if already_filtered else "0"
_prev_defer = os.getenv("GLOSSARY_DEFER_SAVE")
os.environ["GLOSSARY_DEFER_SAVE"] = "1"
try:
result = self._extract_with_custom_prompt(
custom_prompt, chunk_text, language,
min_frequency, max_names, max_titles,
None, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode
)
finally:
os.environ["_CHUNK_ALREADY_FILTERED"] = "0" # Reset
if _prev_defer is None:
if "GLOSSARY_DEFER_SAVE" in os.environ:
del os.environ["GLOSSARY_DEFER_SAVE"]
else:
os.environ["GLOSSARY_DEFER_SAVE"] = _prev_defer
return result
else:
return self._extract_with_patterns(
chunk_text, language, min_frequency,
max_names, max_titles, batch_size,
None, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode
)
def _apply_final_filter(self, entries, filter_mode):
"""Apply final filtering based on mode to ensure only requested types are included"""
if filter_mode == "only_with_honorifics":
# Filter to keep only entries that look like they have honorifics
filtered = {}
for key, value in entries.items():
# Check if the key contains known honorific patterns
if self._has_honorific(key):
filtered[key] = value
print(f"📑 Final filter: Kept {len(filtered)} entries with honorifics (from {len(entries)} total)")
return filtered
elif filter_mode == "only_without_honorifics":
# Filter to keep only entries without honorifics
filtered = {}
for key, value in entries.items():
if not self._has_honorific(key):
filtered[key] = value
print(f"📑 Final filter: Kept {len(filtered)} entries without honorifics (from {len(entries)} total)")
return filtered
else:
return entries
def _looks_like_name(self, text):
"""Check if text looks like a character name"""
if not text:
return False
# Check for various name patterns
# Korean names (2-4 hangul characters)
if all(0xAC00 <= ord(char) <= 0xD7AF for char in text) and 2 <= len(text) <= 4:
return True
# Japanese names (mix of kanji/kana, 2-6 chars)
has_kanji = any(0x4E00 <= ord(char) <= 0x9FFF for char in text)
has_kana = any((0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF) for char in text)
if (has_kanji or has_kana) and 2 <= len(text) <= 6:
return True
# Chinese names (2-4 Chinese characters)
if all(0x4E00 <= ord(char) <= 0x9FFF for char in text) and 2 <= len(text) <= 4:
return True
# English names (starts with capital, mostly letters)
if text[0].isupper() and sum(1 for c in text if c.isalpha()) >= len(text) * 0.8:
return True
return False
def _has_honorific(self, term):
"""Check if a term contains an honorific using PatternManager's comprehensive list"""
if not term:
return False
term_lower = term.lower()
# Check all language honorifics from PatternManager
for language, honorifics_list in self.pattern_manager.CJK_HONORIFICS.items():
for honorific in honorifics_list:
# For romanized/English honorifics with spaces or dashes
if honorific.startswith(' ') or honorific.startswith('-'):
if term_lower.endswith(honorific.lower()):
return True
# For CJK honorifics (no separator)
else:
if honorific in term:
return True
return False
def _strip_all_honorifics(self, term, language='korean'):
"""Strip all honorifics from a term using PatternManager's lists"""
if not term:
return term
result = term
# Get honorifics for the specific language and English romanizations
honorifics_to_strip = []
if language in self.pattern_manager.CJK_HONORIFICS:
honorifics_to_strip.extend(self.pattern_manager.CJK_HONORIFICS[language])
honorifics_to_strip.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
# Sort by length (longest first) to avoid partial matches
honorifics_to_strip.sort(key=len, reverse=True)
# Strip honorifics
for honorific in honorifics_to_strip:
if honorific.startswith(' ') or honorific.startswith('-'):
# For romanized honorifics with separators
if result.lower().endswith(honorific.lower()):
result = result[:-len(honorific)]
else:
# For CJK honorifics (no separator)
if result.endswith(honorific):
result = result[:-len(honorific)]
return result.strip()
def _convert_to_csv_format(self, data):
"""Convert various glossary formats to CSV string format with enforced 3 columns"""
csv_lines = ["type,raw_name,translated_name"]
if isinstance(data, str):
# Already CSV string
if data.strip().startswith('type,raw_name'):
return data
# Try to parse as JSON
try:
data = json.loads(data)
except:
return data
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
if 'type' in item and 'raw_name' in item:
# Already in correct format
line = f"{item['type']},{item['raw_name']},{item.get('translated_name', item['raw_name'])}"
csv_lines.append(line)
else:
# Old format - default to 'term' type
entry_type = 'term'
raw_name = item.get('original_name', '')
translated_name = item.get('name', raw_name)
if raw_name and translated_name:
csv_lines.append(f"{entry_type},{raw_name},{translated_name}")
elif isinstance(data, dict):
if 'entries' in data:
# Has metadata wrapper, extract entries
for original, translated in data['entries'].items():
csv_lines.append(f"term,{original},{translated}")
else:
# Plain dictionary - default to 'term' type
for original, translated in data.items():
csv_lines.append(f"term,{original},{translated}")
return '\n'.join(csv_lines)
def _parse_csv_to_dict(self, csv_content):
"""Parse CSV content to dictionary for backward compatibility"""
result = {}
lines = csv_content.strip().split('\n')
for line in lines[1:]: # Skip header
if not line.strip():
continue
parts = [p.strip() for p in line.split(',')]
if len(parts) >= 3:
result[parts[1]] = parts[2] # raw_name -> translated_name
return result
def _fuzzy_match(self, term1, term2, threshold=0.90):
"""Check if two terms match using fuzzy matching"""
ratio = SequenceMatcher(None, term1.lower(), term2.lower()).ratio()
return ratio >= threshold
def _fuzzy_match_rapidfuzz(self, term_lower, text_lower, threshold, term_len):
"""Use rapidfuzz library for MUCH faster fuzzy matching"""
from rapidfuzz import fuzz
print(f"📑 Using RapidFuzz (C++ speed)...")
start_time = time.time()
matches_count = 0
threshold_percent = threshold * 100 # rapidfuzz uses 0-100 scale
# Can use smaller step because rapidfuzz is so fast
step = 1 # Check every position - rapidfuzz can handle it
# Process text
for i in range(0, len(text_lower) - term_len + 1, step):
# Check stop flag every 10000 positions
if i > 0 and i % 10000 == 0:
if is_stop_requested():
print(f"📑 RapidFuzz stopped at position {i}")
return matches_count
window = text_lower[i:i + term_len]
# rapidfuzz is fast enough we can check every position
if fuzz.ratio(term_lower, window) >= threshold_percent:
matches_count += 1
elapsed = time.time() - start_time
print(f"📑 RapidFuzz found {matches_count} matches in {elapsed:.2f}s")
return matches_count
def _batch_compute_frequencies(self, terms, all_text, fuzzy_threshold=0.90, min_frequency=2):
"""Compute frequencies for all terms at once - MUCH faster than individual checking"""
print(f"📑 Computing frequencies for {len(terms)} terms in batch mode...")
start_time = time.time()
# Result dictionary
term_frequencies = {}
# First pass: exact matching (very fast)
print(f"📑 Phase 1: Exact matching...")
text_lower = all_text.lower()
for term in terms:
if is_stop_requested():
return term_frequencies
term_lower = term.lower()
count = text_lower.count(term_lower)
term_frequencies[term] = count
exact_time = time.time() - start_time
high_freq_terms = sum(1 for count in term_frequencies.values() if count >= min_frequency)
print(f"📑 Exact matching complete: {high_freq_terms}/{len(terms)} terms meet threshold ({exact_time:.1f}s)")
# If fuzzy matching is disabled, we're done
if fuzzy_threshold >= 1.0:
return term_frequencies
# Second pass: fuzzy matching ONLY for low-frequency terms
low_freq_terms = [term for term, count in term_frequencies.items() if count < min_frequency]
if low_freq_terms:
print(f"📑 Phase 2: Fuzzy matching for {len(low_freq_terms)} low-frequency terms...")
# Try to use RapidFuzz batch processing
try:
from rapidfuzz import process, fuzz
# For very large texts, sample it for fuzzy matching
if len(text_lower) > 500000:
print(f"📑 Text too large ({len(text_lower):,} chars), sampling for fuzzy matching...")
# Sample every Nth character to reduce size
sample_rate = max(1, len(text_lower) // 100000)
sampled_text = text_lower[::sample_rate]
else:
sampled_text = text_lower
# Create chunks of text for fuzzy matching
chunk_size = 1000 # Process text in chunks
text_chunks = [sampled_text[i:i+chunk_size] for i in range(0, len(sampled_text), chunk_size//2)] # Overlapping chunks
print(f"📑 Processing {len(text_chunks)} text chunks...")
threshold_percent = fuzzy_threshold * 100
# Process in batches to avoid memory issues
batch_size = 100 # Process 100 terms at a time
for batch_start in range(0, len(low_freq_terms), batch_size):
if is_stop_requested():
break
batch_end = min(batch_start + batch_size, len(low_freq_terms))
batch_terms = low_freq_terms[batch_start:batch_end]
for term in batch_terms:
if is_stop_requested():
break
# Quick fuzzy search in chunks
fuzzy_count = 0
for chunk in text_chunks[:50]: # Limit to first 50 chunks for speed
if fuzz.partial_ratio(term.lower(), chunk) >= threshold_percent:
fuzzy_count += 1
if fuzzy_count > 0:
# Scale up based on sampling
if len(text_lower) > 500000:
fuzzy_count *= (len(text_lower) // len(sampled_text))
term_frequencies[term] += fuzzy_count
if (batch_end % 500 == 0) or (batch_end == len(low_freq_terms)):
elapsed = time.time() - start_time
print(f"📑 Processed {batch_end}/{len(low_freq_terms)} terms ({elapsed:.1f}s)")
except ImportError:
print("📑 RapidFuzz not available, skipping fuzzy matching")
total_time = time.time() - start_time
final_high_freq = sum(1 for count in term_frequencies.values() if count >= min_frequency)
print(f"📑 Batch frequency computation complete: {final_high_freq}/{len(terms)} terms accepted ({total_time:.1f}s)")
return term_frequencies
def _find_fuzzy_matches(self, term, text, threshold=0.90):
"""Find fuzzy matches of a term in text using efficient method with parallel processing"""
start_time = time.time()
term_lower = term.lower()
text_lower = text.lower()
term_len = len(term)
# Only log for debugging if explicitly enabled
debug_search = os.getenv("GLOSSARY_DEBUG_SEARCH", "0") == "1"
if debug_search and len(text) > 100000:
print(f"📑 Searching for '{term}' in {len(text):,} chars (threshold: {threshold})")
# Strategy 1: Use exact matching first for efficiency
exact_start = time.time()
matches_count = text_lower.count(term_lower)
exact_time = time.time() - exact_start
if matches_count > 0:
if debug_search and len(text) > 100000:
print(f"📑 Found {matches_count} exact matches in {exact_time:.3f}s")
return matches_count
# Strategy 2: Try rapidfuzz if available (much faster)
if matches_count == 0 and threshold < 1.0:
try:
from rapidfuzz import fuzz
return self._fuzzy_match_rapidfuzz(term_lower, text_lower, threshold, term_len)
except ImportError:
pass # Fall back to parallel/sequential
# Strategy 3: Fall back to parallel/sequential if rapidfuzz not available
# Check if parallel processing is enabled
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
if extraction_workers > 1 and len(text) > 50000: # Use parallel for large texts
return self._parallel_fuzzy_search(term_lower, text_lower, threshold, term_len, extraction_workers)
else:
return self._sequential_fuzzy_search(term_lower, text_lower, threshold, term_len)
# Check if parallel processing is enabled
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
if extraction_workers > 1 and len(text) > 50000: # Use parallel for large texts
return self._parallel_fuzzy_search(term_lower, text_lower, threshold, term_len, extraction_workers)
else:
return self._sequential_fuzzy_search(term_lower, text_lower, threshold, term_len)
return matches_count
def _parallel_fuzzy_search(self, term_lower, text_lower, threshold, term_len, num_workers):
"""Parallel fuzzy search using ThreadPoolExecutor"""
print(f"📑 Starting parallel fuzzy search with {num_workers} workers...")
text_len = len(text_lower)
matches_count = 0
# Split text into overlapping chunks for parallel processing
chunk_size = max(text_len // num_workers, term_len * 100)
chunks = []
for i in range(0, text_len, chunk_size):
# Add overlap to avoid missing matches at boundaries
end = min(i + chunk_size + term_len - 1, text_len)
chunks.append((i, text_lower[i:end]))
print(f"📑 Split into {len(chunks)} chunks of ~{chunk_size:,} chars each")
# Process chunks in parallel
with ThreadPoolExecutor(max_workers=num_workers) as executor:
futures = []
for chunk_idx, (start_pos, chunk_text) in enumerate(chunks):
if is_stop_requested():
return matches_count
future = executor.submit(
self._fuzzy_search_chunk,
term_lower, chunk_text, threshold, term_len, chunk_idx, len(chunks)
)
futures.append(future)
# Collect results
for future in as_completed(futures):
if is_stop_requested():
executor.shutdown(wait=False)
return matches_count
try:
chunk_matches = future.result()
matches_count += chunk_matches
except Exception as e:
print(f"📑 ⚠️ Chunk processing error: {e}")
print(f"📑 Parallel fuzzy search found {matches_count} matches")
return matches_count
def _fuzzy_search_chunk(self, term_lower, chunk_text, threshold, term_len, chunk_idx, total_chunks):
"""Process a single chunk for fuzzy matches"""
chunk_matches = 0
# Use a more efficient step size - no need to check every position
step = max(1, term_len // 3) # Check every third of term length
for i in range(0, len(chunk_text) - term_len + 1, step):
# Check stop flag periodically
if i > 0 and i % 1000 == 0:
if is_stop_requested():
return chunk_matches
window = chunk_text[i:i + term_len]
# Use SequenceMatcher for fuzzy matching
if SequenceMatcher(None, term_lower, window).ratio() >= threshold:
chunk_matches += 1
# Log progress for this chunk
if total_chunks > 1:
print(f"📑 Chunk {chunk_idx + 1}/{total_chunks} completed: {chunk_matches} matches")
return chunk_matches
def _sequential_fuzzy_search(self, term_lower, text_lower, threshold, term_len):
"""Sequential fuzzy search (fallback for small texts or single worker)"""
print(f"📑 Starting sequential fuzzy search...")
fuzzy_start = time.time()
matches_count = 0
# More efficient step size
step = max(1, term_len // 3)
total_windows = (len(text_lower) - term_len + 1) // step
print(f"📑 Checking ~{total_windows:,} windows with step size {step}")
windows_checked = 0
for i in range(0, len(text_lower) - term_len + 1, step):
# Check stop flag frequently
if i > 0 and i % (step * 100) == 0:
if is_stop_requested():
return matches_count
# Progress log for very long operations
if windows_checked % 1000 == 0 and windows_checked > 0:
elapsed = time.time() - fuzzy_start
rate = windows_checked / elapsed if elapsed > 0 else 0
eta = (total_windows - windows_checked) / rate if rate > 0 else 0
print(f"📑 Progress: {windows_checked}/{total_windows} windows, {rate:.0f} w/s, ETA: {eta:.1f}s")
window = text_lower[i:i + term_len]
if SequenceMatcher(None, term_lower, window).ratio() >= threshold:
matches_count += 1
windows_checked += 1
fuzzy_time = time.time() - fuzzy_start
print(f"📑 Sequential fuzzy search completed in {fuzzy_time:.2f}s, found {matches_count} matches")
return matches_count
def _fuzzy_match(self, term1, term2, threshold=0.90):
"""Check if two terms match using fuzzy matching (unchanged)"""
ratio = SequenceMatcher(None, term1.lower(), term2.lower()).ratio()
return ratio >= threshold
def _strip_honorific(self, term, language_hint='unknown'):
"""Strip honorific from a term if present"""
if not term:
return term
# Get honorifics for the detected language
honorifics_to_check = []
if language_hint in self.pattern_manager.CJK_HONORIFICS:
honorifics_to_check.extend(self.pattern_manager.CJK_HONORIFICS[language_hint])
honorifics_to_check.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
# Check and remove honorifics
for honorific in honorifics_to_check:
if honorific.startswith('-') or honorific.startswith(' '):
# English-style suffix
if term.endswith(honorific):
return term[:-len(honorific)].strip()
else:
# CJK-style suffix (no separator)
if term.endswith(honorific):
return term[:-len(honorific)]
return term
def _translate_chunk_traditional(self, chunk_text, chunk_index, total_chunks, chapter_title=""):
"""Simplified translation for traditional APIs (DeepL, Google Translate)"""
print(f"📝 Using traditional translation API for chunk {chunk_index}/{total_chunks}")
# Traditional APIs don't use complex prompts, just need the text
messages = []
# Add minimal system context for language detection
profile = self.active_profile
if profile == 'korean':
lang_hint = "Translating from Korean to English"
elif profile == 'japanese':
lang_hint = "Translating from Japanese to English"
elif profile == 'chinese':
lang_hint = "Translating from Chinese to English"
else:
lang_hint = "Translating to English"
messages.append({
"role": "system",
"content": lang_hint
})
# For traditional APIs, we need to handle glossary differently
# Apply glossary terms as preprocessing if available
processed_text = chunk_text
if hasattr(self, 'glossary_manager') and self.glossary_manager and self.glossary_manager.entries:
# Pre-process: Mark glossary terms with placeholders
glossary_placeholders = {}
placeholder_index = 0
for entry in self.glossary_manager.entries:
source = entry.get('source', '')
target = entry.get('target', '')
if source and target and source in processed_text:
# Create unique placeholder
placeholder = f"[[GLOSS_{placeholder_index}]]"
glossary_placeholders[placeholder] = target
processed_text = processed_text.replace(source, placeholder)
placeholder_index += 1
print(f"📚 Applied {len(glossary_placeholders)} glossary placeholders")
# Add the text to translate
messages.append({
"role": "user",
"content": processed_text
})
# Send to API
try:
response = self.client.send(messages)
if response and response.content:
translated_text = response.content
# Post-process: Replace placeholders with glossary terms
if 'glossary_placeholders' in locals():
for placeholder, target in glossary_placeholders.items():
translated_text = translated_text.replace(placeholder, target)
print(f"✅ Restored {len(glossary_placeholders)} glossary terms")
# Log detected language if available
if hasattr(response, 'usage') and response.usage:
detected_lang = response.usage.get('detected_source_lang')
if detected_lang:
print(f"🔍 Detected source language: {detected_lang}")
return translated_text
else:
print("❌ No translation received from traditional API")
return None
except Exception as e:
print(f"❌ Traditional API translation error: {e}")
return None
def _filter_text_for_glossary(self, text, min_frequency=2):
"""Filter text to extract only meaningful content for glossary extraction"""
import re
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
filter_start_time = time.time()
print(f"📑 Starting smart text filtering...")
print(f"📑 Input text size: {len(text):,} characters")
# Clean HTML if present
print(f"📑 Step 1/7: Cleaning HTML tags...")
from bs4 import BeautifulSoup
soup = BeautifulSoup(text, 'html.parser')
clean_text = soup.get_text()
print(f"📑 Clean text size: {len(clean_text):,} characters")
# Detect primary language for better filtering
print(f"📑 Step 2/7: Detecting primary language...")
def detect_primary_language(text_sample):
sample = text_sample[:1000]
korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
if korean_chars > 50:
return 'korean'
elif japanese_kana > 20:
return 'japanese'
elif chinese_chars > 50 and japanese_kana < 10:
return 'chinese'
else:
return 'english'
primary_lang = detect_primary_language(clean_text)
print(f"📑 Detected primary language: {primary_lang}")
# Split into sentences for better context
print(f"📑 Step 3/7: Splitting text into sentences...")
sentences = re.split(r'[.!?。！？]+', clean_text)
print(f"📑 Found {len(sentences):,} sentences")
# Extract potential terms (words/phrases that appear multiple times)
print(f"📑 Step 4/7: Setting up extraction patterns and exclusion rules...")
word_freq = Counter()
# Pattern for detecting potential names/terms based on capitalization or special characters
# Korean names: 2-4 hangul characters WITHOUT honorifics
korean_pattern = r'[가-힣]{2,4}'
# Japanese names: kanji/hiragana/katakana combinations
japanese_pattern = r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]{2,6}'
# Chinese names: 2-4 Chinese characters
chinese_pattern = r'[\u4e00-\u9fff]{2,4}'
# English proper nouns: Capitalized words
english_pattern = r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b'
# Combine patterns
combined_pattern = f'({korean_pattern}|{japanese_pattern}|{chinese_pattern}|{english_pattern})'
print(f"📑 Using combined regex pattern for {primary_lang} text")
# Get honorifics and title patterns for the detected language
honorifics_to_exclude = set()
if primary_lang in self.pattern_manager.CJK_HONORIFICS:
honorifics_to_exclude.update(self.pattern_manager.CJK_HONORIFICS[primary_lang])
# Also add English romanizations
honorifics_to_exclude.update(self.pattern_manager.CJK_HONORIFICS.get('english', []))
# Compile title patterns for the language
title_patterns = []
if primary_lang in self.pattern_manager.TITLE_PATTERNS:
for pattern in self.pattern_manager.TITLE_PATTERNS[primary_lang]:
title_patterns.append(re.compile(pattern))
# Function to check if a term should be excluded
def should_exclude_term(term):
term_lower = term.lower()
# Check if it's a common word
if term in self.pattern_manager.COMMON_WORDS or term_lower in self.pattern_manager.COMMON_WORDS:
return True
# Check if it contains honorifics
for honorific in honorifics_to_exclude:
if honorific in term or (honorific.startswith('-') and term.endswith(honorific[1:])):
return True
# Check if it matches title patterns
for pattern in title_patterns:
if pattern.search(term):
return True
# Check if it's a number (including Chinese numbers)
if term in self.pattern_manager.CHINESE_NUMS:
return True
# Check if it's just digits
if term.isdigit():
return True
return False
# Extract potential terms from each sentence
print(f"📑 Step 5/7: Extracting and filtering terms from sentences...")
# Check if we should use parallel processing
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
# Auto-detect optimal workers if not set
if extraction_workers == 1 and len(sentences) > 1000:
# Use more cores for better parallelization
cpu_count = os.cpu_count() or 4
extraction_workers = min(cpu_count, 12) # Use up to 12 cores
print(f"📑 Auto-detected {cpu_count} CPU cores, using {extraction_workers} workers")
use_parallel = extraction_workers > 1 and len(sentences) > 100
if use_parallel:
print(f"📑 Using parallel processing with {extraction_workers} workers")
print(f"📑 Estimated speedup: {extraction_workers}x faster")
important_sentences = []
seen_contexts = set()
processed_count = 0
total_sentences = len(sentences)
last_progress_time = time.time()
def process_sentence_batch(batch_sentences, batch_idx):
"""Process a batch of sentences"""
local_word_freq = Counter()
local_important = []
local_seen = set()
for sentence in batch_sentences:
sentence = sentence.strip()
if len(sentence) < 10 or len(sentence) > 500:
continue
# Find all potential terms in this sentence
matches = re.findall(combined_pattern, sentence)
if matches:
# Filter out excluded terms
filtered_matches = []
for match in matches:
if not should_exclude_term(match):
local_word_freq[match] += 1
filtered_matches.append(match)
# Keep sentences with valid potential terms
if filtered_matches:
sentence_key = ' '.join(sorted(filtered_matches))
if sentence_key not in local_seen:
local_important.append(sentence)
local_seen.add(sentence_key)
return local_word_freq, local_important, local_seen, batch_idx
if use_parallel:
# Force SMALL batches for real parallelization
# We want MANY small batches, not few large ones!
# Calculate based on total sentences
total_sentences = len(sentences)
if total_sentences < 1000:
# Small dataset: 50-100 sentences per batch
optimal_batch_size = 100
elif total_sentences < 10000:
# Medium dataset: 200 sentences per batch
optimal_batch_size = 200
elif total_sentences < 50000:
# Large dataset: 300 sentences per batch
optimal_batch_size = 300
else:
# Very large dataset: 400 sentences per batch max
optimal_batch_size = 400
# Ensure we have enough batches for all workers
min_batches = extraction_workers * 3 # At least 3 batches per worker
max_batch_size = max(50, total_sentences // min_batches)
optimal_batch_size = min(optimal_batch_size, max_batch_size)
print(f"📑 Total sentences: {total_sentences:,}")
print(f"📑 Target batch size: {optimal_batch_size} sentences")
# Calculate expected number of batches
expected_batches = (total_sentences + optimal_batch_size - 1) // optimal_batch_size
print(f"📑 Expected batches: {expected_batches} (for {extraction_workers} workers)")
print(f"📑 Batches per worker: ~{expected_batches // extraction_workers} batches")
batches = [sentences[i:i + optimal_batch_size] for i in range(0, len(sentences), optimal_batch_size)]
print(f"📑 Processing {len(batches)} batches of ~{optimal_batch_size} sentences each")
print(f"📑 Expected speedup: {min(extraction_workers, len(batches))}x (using {extraction_workers} workers)")
# Decide between ThreadPoolExecutor and ProcessPoolExecutor
import multiprocessing
in_subprocess = multiprocessing.current_process().name != 'MainProcess'
# Use ProcessPoolExecutor for better parallelism on larger datasets
use_process_pool = (not in_subprocess and len(sentences) > 5000)
if use_process_pool:
print(f"📑 Using ProcessPoolExecutor for maximum performance (true parallelism)")
executor_class = ProcessPoolExecutor
else:
print(f"📑 Using ThreadPoolExecutor for sentence processing")
executor_class = ThreadPoolExecutor
with executor_class(max_workers=extraction_workers) as executor:
futures = []
# Prepare data for ProcessPoolExecutor if needed
if use_process_pool:
# Serialize exclusion check data for process pool
exclude_check_data = (
list(honorifics_to_exclude),
[p.pattern for p in title_patterns], # Convert regex to strings
self.pattern_manager.COMMON_WORDS,
self.pattern_manager.CHINESE_NUMS
)
for idx, batch in enumerate(batches):
if use_process_pool:
# Use module-level function for ProcessPoolExecutor
future = executor.submit(_process_sentence_batch_for_extraction,
(batch, idx, combined_pattern, exclude_check_data))
else:
# Use local function for ThreadPoolExecutor
future = executor.submit(process_sentence_batch, batch, idx)
futures.append(future)
# Yield to GUI when submitting futures
if idx % 10 == 0:
time.sleep(0.001)
# Collect results with progress
completed_batches = 0
batch_start_time = time.time()
for future in as_completed(futures):
# Get result without timeout - as_completed already handles waiting
local_word_freq, local_important, local_seen, batch_idx = future.result()
# Merge results
word_freq.update(local_word_freq)
for sentence in local_important:
sentence_key = ' '.join(sorted(re.findall(combined_pattern, sentence)))
if sentence_key not in seen_contexts:
important_sentences.append(sentence)
seen_contexts.add(sentence_key)
processed_count += len(batches[batch_idx])
completed_batches += 1
# Show progress every 10 batches or at key milestones
if completed_batches % 10 == 0 or completed_batches == len(batches):
progress = (processed_count / total_sentences) * 100
elapsed = time.time() - batch_start_time
rate = (processed_count / elapsed) if elapsed > 0 else 0
print(f"📑 Progress: {processed_count:,}/{total_sentences:,} sentences ({progress:.1f}%) | Batch {completed_batches}/{len(batches)} | {rate:.0f} sent/sec")
# Yield to GUI after each batch completes
time.sleep(0.001)
else:
# Sequential processing with progress
for idx, sentence in enumerate(sentences):
sentence = sentence.strip()
if len(sentence) < 10 or len(sentence) > 500:
continue
# Find all potential terms in this sentence
matches = re.findall(combined_pattern, sentence)
if matches:
# Filter out excluded terms
filtered_matches = []
for match in matches:
if not should_exclude_term(match):
word_freq[match] += 1
filtered_matches.append(match)
# Keep sentences with valid potential terms
if filtered_matches:
sentence_key = ' '.join(sorted(filtered_matches))
if sentence_key not in seen_contexts:
important_sentences.append(sentence)
seen_contexts.add(sentence_key)
# Show progress every 1000 sentences or 2 seconds
if idx % 1000 == 0 or (time.time() - last_progress_time > 2):
progress = ((idx + 1) / total_sentences) * 100
print(f"📑 Processing sentences: {idx + 1:,}/{total_sentences:,} ({progress:.1f}%)")
last_progress_time = time.time()
# Yield to GUI thread every 1000 sentences
time.sleep(0.001) # Tiny sleep to let GUI update
# Yield to GUI thread every 1000 sentences
time.sleep(0.001) # Tiny sleep to let GUI update
print(f"📑 Found {len(important_sentences):,} sentences with potential glossary terms")
# Step 6/7: Deduplicate and normalize terms
print(f"📑 Step 6/7: Normalizing and deduplicating {len(word_freq):,} unique terms...")
# Since should_exclude_term already filters honorifics, we just need to deduplicate
# based on normalized forms (lowercase, etc.)
combined_freq = Counter()
term_count = 0
for term, count in word_freq.items():
# Normalize term for deduplication (but keep original form)
normalized = term.lower().strip()
# Keep the version with highest count
if normalized in combined_freq:
# If we already have this normalized form, keep the one with higher count
if count > combined_freq[normalized]:
# Remove old entry and add new one
del combined_freq[normalized]
combined_freq[term] = count
else:
combined_freq[term] = count
term_count += 1
# Yield to GUI every 1000 terms
if term_count % 1000 == 0:
time.sleep(0.001)
print(f"📑 Deduplicated to {len(combined_freq):,} unique terms")
# Filter to keep only terms that appear at least min_frequency times
frequent_terms = {term: count for term, count in combined_freq.items() if count >= min_frequency}
# Build filtered text focusing on sentences containing frequent terms
print(f"📑 Step 7/7: Building filtered text from relevant sentences...")
# OPTIMIZATION: Skip sentences that already passed filtering in step 5
# These sentences already contain glossary terms, no need to check again!
# We just need to limit the sample size
filtered_sentences = important_sentences # Already filtered!
print(f"📑 Using {len(filtered_sentences):,} pre-filtered sentences (already contain glossary terms)")
# For extremely large datasets, we can optionally do additional filtering
if len(filtered_sentences) > 10000 and len(frequent_terms) > 1000:
print(f"📑 Large dataset detected - applying frequency-based filtering...")
print(f"📑 Filtering {len(filtered_sentences):,} sentences for top frequent terms...")
# Sort terms by frequency to prioritize high-frequency ones
sorted_terms = sorted(frequent_terms.items(), key=lambda x: x[1], reverse=True)
top_terms = dict(sorted_terms[:1000]) # Focus on top 1000 most frequent terms
print(f"📑 Using top {len(top_terms):,} most frequent terms for final filtering")
# Use parallel processing only if really needed
if use_parallel and len(filtered_sentences) > 5000:
import multiprocessing
in_subprocess = multiprocessing.current_process().name != 'MainProcess'
# Create a simple set of terms for fast lookup (no variations needed)
term_set = set(top_terms.keys())
print(f"📑 Using parallel filtering with {extraction_workers} workers...")
# Optimize batch size
check_batch_size = 500 # Larger batches since we're doing simpler checks
check_batches = [filtered_sentences[i:i + check_batch_size]
for i in range(0, len(filtered_sentences), check_batch_size)]
print(f"📑 Processing {len(check_batches)} batches of ~{check_batch_size} sentences")
# Simple function to check if sentence contains any top term
def check_batch_simple(batch):
result = []
for sentence in batch:
# Simple substring check - much faster than regex
for term in term_set:
if term in sentence:
result.append(sentence)
break
return result
new_filtered = []
with ThreadPoolExecutor(max_workers=extraction_workers) as executor:
futures = [executor.submit(check_batch_simple, batch) for batch in check_batches]
for future in as_completed(futures):
new_filtered.extend(future.result())
filtered_sentences = new_filtered
print(f"📑 Filtered to {len(filtered_sentences):,} sentences containing top terms")
else:
# For smaller datasets, simple sequential filtering
print(f"📑 Using sequential filtering...")
new_filtered = []
for i, sentence in enumerate(filtered_sentences):
for term in top_terms:
if term in sentence:
new_filtered.append(sentence)
break
if i % 1000 == 0:
print(f"📑 Progress: {i:,}/{len(filtered_sentences):,} sentences")
time.sleep(0.001)
filtered_sentences = new_filtered
print(f"📑 Filtered to {len(filtered_sentences):,} sentences containing top terms")
print(f"📑 Selected {len(filtered_sentences):,} sentences containing frequent terms")
# Limit the number of sentences to reduce token usage
max_sentences = int(os.getenv("GLOSSARY_MAX_SENTENCES", "200"))
if len(filtered_sentences) > max_sentences:
print(f"📑 Limiting to {max_sentences} representative sentences (from {len(filtered_sentences):,})")
# Take a representative sample
step = len(filtered_sentences) // max_sentences
filtered_sentences = filtered_sentences[::step][:max_sentences]
filtered_text = ' '.join(filtered_sentences)
# Calculate and display filtering statistics
filter_end_time = time.time()
filter_duration = filter_end_time - filter_start_time
original_length = len(clean_text)
filtered_length = len(filtered_text)
reduction_percent = ((original_length - filtered_length) / original_length * 100) if original_length > 0 else 0
print(f"\n📑 === FILTERING COMPLETE ===")
print(f"📑 Duration: {filter_duration:.1f} seconds")
print(f"📑 Text reduction: {original_length:,} → {filtered_length:,} chars ({reduction_percent:.1f}% reduction)")
print(f"📑 Terms found: {len(frequent_terms):,} unique terms (min frequency: {min_frequency})")
print(f"📑 Final output: {len(filtered_sentences)} sentences, {filtered_length:,} characters")
print(f"📑 Performance: {(original_length / filter_duration / 1000):.1f}K chars/second")
print(f"📑 ========================\n")
return filtered_text, frequent_terms
def _extract_with_custom_prompt(self, custom_prompt, all_text, language,
min_frequency, max_names, max_titles,
existing_glossary, output_dir,
strip_honorifics=True, fuzzy_threshold=0.90, filter_mode='all'):
"""Extract glossary using custom AI prompt with proper filtering"""
print("📑 Using custom automatic glossary prompt")
extraction_start = time.time()
# Check stop flag
if is_stop_requested():
print("📑 ❌ Glossary extraction stopped by user")
return {}
# Note: Filter mode can be controlled via the configurable prompt environment variable
# No hardcoded filter instructions are added here
try:
MODEL = os.getenv("MODEL", "gemini-2.0-flash")
API_KEY = (os.getenv("API_KEY") or
os.getenv("OPENAI_API_KEY") or
os.getenv("OPENAI_OR_Gemini_API_KEY") or
os.getenv("GEMINI_API_KEY"))
if is_traditional_translation_api(MODEL):
return self._translate_chunk_traditional(chunk_text, chunk_index, total_chunks, chapter_title)
elif not API_KEY:
print(f"📑 No API key found, falling back to pattern-based extraction")
return self._extract_with_patterns(all_text, language, min_frequency,
max_names, max_titles, 50,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
else:
print(f"📑 Using AI-assisted extraction with custom prompt")
from unified_api_client import UnifiedClient, UnifiedClientError
client = UnifiedClient(model=MODEL, api_key=API_KEY, output_dir=output_dir)
if hasattr(client, 'reset_cleanup_state'):
client.reset_cleanup_state()
# Apply thread submission delay using the client's method
thread_delay = float(os.getenv("THREAD_SUBMISSION_DELAY_SECONDS", "0.5"))
if thread_delay > 0:
client._apply_thread_submission_delay()
# Check if cancelled during delay
if hasattr(client, '_cancelled') and client._cancelled:
print("📑 ❌ Glossary extraction stopped during delay")
return {}
# Check if text is already filtered (from chunking)
already_filtered = os.getenv("_CHUNK_ALREADY_FILTERED", "0") == "1"
if already_filtered:
print("📑 Text already filtered during chunking, skipping re-filtering")
text_sample = all_text # Use as-is since it's already filtered
detected_terms = {}
else:
# Apply smart filtering to reduce noise and focus on meaningful content
force_disable = os.getenv("GLOSSARY_FORCE_DISABLE_SMART_FILTER", "0") == "1"
use_smart_filter = (os.getenv("GLOSSARY_USE_SMART_FILTER", "1") == "1") and not force_disable
if use_smart_filter:
print("📑 Applying smart text filtering to reduce noise...")
text_sample, detected_terms = self._filter_text_for_glossary(all_text, min_frequency)
else:
print("📑 Smart filter disabled - using raw text sample")
# Fallback to simple truncation
max_text_size = int(os.getenv("GLOSSARY_MAX_TEXT_SIZE", "50000"))
text_sample = all_text[:max_text_size] if len(all_text) > max_text_size and max_text_size > 0 else all_text
detected_terms = {}
# Replace placeholders in prompt
prompt = custom_prompt.replace('{language}', language)
prompt = prompt.replace('{min_frequency}', str(min_frequency))
prompt = prompt.replace('{max_names}', str(max_names))
prompt = prompt.replace('{max_titles}', str(max_titles))
# Get the format instructions from environment variable
format_instructions = os.getenv("GLOSSARY_FORMAT_INSTRUCTIONS", "")
# If no format instructions are provided, use a default
if not format_instructions:
format_instructions = """
Return the results in EXACT CSV format with this header:
type,raw_name,translated_name
For example:
character,김상현,Kim Sang-hyu
character,갈편제,Gale Hardest
character,디히릿 아데,Dihirit Ade
Only include entries that actually appear in the text.
Do not use quotes around values unless they contain commas.
Text to analyze:
{text_sample}"""
# Replace placeholders in format instructions
format_instructions = format_instructions.replace('{text_sample}', text_sample)
# Combine the user's prompt with format instructions
enhanced_prompt = f"{prompt}\n\n{format_instructions}"
messages = [
{"role": "system", "content": "You are a glossary extraction assistant. Return ONLY CSV format with exactly 3 columns: type,raw_name,translated_name. The 'type' column should classify entries (e.g., character, term, location, etc.)."},
{"role": "user", "content": enhanced_prompt}
]
# Check stop before API call
if is_stop_requested():
print("📑 ❌ Glossary extraction stopped before API call")
return {}
try:
temperature = float(os.getenv("TEMPERATURE", "0.3"))
max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "4096"))
# Use send_with_interrupt for interruptible API call
chunk_timeout = int(os.getenv("CHUNK_TIMEOUT", "900")) # 15 minute default for glossary
print(f"📑 Sending AI extraction request (timeout: {chunk_timeout}s, interruptible)...")
# Before API call
api_start = time.time()
print(f"📑 Preparing API request (text size: {len(text_sample):,} chars)...")
print(f"📑 ⏳ Processing {len(text_sample):,} characters... Please wait, this may take 5-10 minutes")
response = send_with_interrupt(
messages=messages,
client=client,
temperature=temperature,
max_tokens=max_tokens,
stop_check_fn=is_stop_requested,
chunk_timeout=chunk_timeout
)
api_time = time.time() - api_start
print(f"📑 API call completed in {api_time:.1f}s")
# Get the actual text from the response
if hasattr(response, 'content'):
response_text = response.content
else:
response_text = str(response)
# Before processing response
process_start = time.time()
print(f"📑 Processing AI response...")
# Process response and build CSV
csv_lines = self._process_ai_response(response_text, all_text, min_frequency,
strip_honorifics, fuzzy_threshold,
language, filter_mode)
print(f"📑 AI extracted {len(csv_lines) - 1} valid terms (header excluded)")
process_time = time.time() - process_start
print(f"📑 Response processing took {process_time:.1f}s")
# If we're running per-chunk, defer all heavy work and saving
if os.getenv("GLOSSARY_DEFER_SAVE", "0") == "1":
return csv_lines
# Check stop before merging
if is_stop_requested():
print("📑 ❌ Glossary generation stopped before merging")
return {}
# Merge with existing glossary if present
if existing_glossary:
csv_lines = self._merge_csv_entries(csv_lines, existing_glossary, strip_honorifics, language)
# Fuzzy matching deduplication
skip_frequency_check = os.getenv("GLOSSARY_SKIP_FREQUENCY_CHECK", "0") == "1"
if not skip_frequency_check: # Only dedupe if we're checking frequencies
# Time the deduplication
dedup_start = time.time()
original_count = len(csv_lines) - 1 # Exclude header
csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
dedup_time = time.time() - dedup_start
final_count = len(csv_lines) - 1 # Exclude header
removed_count = original_count - final_count
print(f"📑 Deduplication completed in {dedup_time:.1f}s")
print(f"📑 - Original entries: {original_count}")
print(f"📑 - Duplicates removed: {removed_count}")
print(f"📑 - Final entries: {final_count}")
# Store for summary statistics
self._dedup_time = getattr(self, '_dedup_time', 0) + dedup_time
else:
print(f"📑 Skipping deduplication (frequency check disabled)")
# Apply filter mode to final results
csv_lines = self._filter_csv_by_mode(csv_lines, filter_mode)
# Check if we should use token-efficient format
use_legacy_format = os.getenv('GLOSSARY_USE_LEGACY_CSV', '0') == '1'
if not use_legacy_format:
# Convert to token-efficient format
csv_lines = self._convert_to_token_efficient_format(csv_lines)
# Final sanitize to prevent stray headers
csv_lines = self._sanitize_final_glossary_lines(csv_lines, use_legacy_format)
# Create final CSV content
csv_content = '\n'.join(csv_lines)
# Save glossary as CSV with proper extension
glossary_path = os.path.join(output_dir, "glossary.csv")
self._atomic_write_file(glossary_path, csv_content)
print(f"\n📑 ✅ AI-ASSISTED GLOSSARY SAVED!")
print(f"📑 File: {glossary_path}")
c_count, t_count, total = self._count_glossary_entries(csv_lines, use_legacy_format)
print(f"📑 Character entries: {c_count}")
print(f"📑 Term entries: {t_count}")
print(f"📑 Total entries: {total}")
total_time = time.time() - extraction_start
print(f"📑 Total extraction time: {total_time:.1f}s")
return self._parse_csv_to_dict(csv_content)
except UnifiedClientError as e:
if "stopped by user" in str(e).lower():
print(f"📑 ❌ AI extraction interrupted by user")
return {}
else:
print(f"⚠️ AI extraction failed: {e}")
print("📑 Falling back to pattern-based extraction")
return self._extract_with_patterns(all_text, language, min_frequency,
max_names, max_titles, 50,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
except Exception as e:
print(f"⚠️ AI extraction failed: {e}")
import traceback
traceback.print_exc()
print("📑 Falling back to pattern-based extraction")
return self._extract_with_patterns(all_text, language, min_frequency,
max_names, max_titles, 50,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
except Exception as e:
print(f"⚠️ Custom prompt processing failed: {e}")
import traceback
traceback.print_exc()
return self._extract_with_patterns(all_text, language, min_frequency,
max_names, max_titles, 50,
existing_glossary, output_dir,
strip_honorifics, fuzzy_threshold, filter_mode)
def _filter_csv_by_mode(self, csv_lines, filter_mode):
"""Filter CSV lines based on the filter mode"""
if filter_mode == "all":
return csv_lines
filtered = [csv_lines[0]] # Keep header
for line in csv_lines[1:]:
if not line.strip():
continue
parts = [p.strip() for p in line.split(',')]
if len(parts) < 3:
continue
entry_type = parts[0].lower()
raw_name = parts[1]
if filter_mode == "only_with_honorifics":
# Only keep character entries with honorifics
if entry_type == "character" and self._has_honorific(raw_name):
filtered.append(line)
elif filter_mode == "only_without_honorifics":
# Keep terms and characters without honorifics
if entry_type == "term" or (entry_type == "character" and not self._has_honorific(raw_name)):
filtered.append(line)
print(f"📑 Filter '{filter_mode}': {len(filtered)-1} entries kept from {len(csv_lines)-1}")
return filtered
def _process_ai_response(self, response_text, all_text, min_frequency,
strip_honorifics, fuzzy_threshold, language, filter_mode):
"""Process AI response and return CSV lines"""
# option to completely skip frequency validation for speed
skip_all_validation = os.getenv("GLOSSARY_SKIP_ALL_VALIDATION", "0") == "1"
if skip_all_validation:
print("📑 ⚡ FAST MODE: Skipping all frequency validation (accepting all AI results)")
# Clean response text
response_text = response_text.strip()
# Remove string representation artifacts if they wrap the entire response
if response_text.startswith('("') and response_text.endswith('")'):
response_text = response_text[2:-2]
elif response_text.startswith('"') and response_text.endswith('"'):
response_text = response_text[1:-1]
elif response_text.startswith('(') and response_text.endswith(')'):
response_text = response_text[1:-1]
# Unescape the string
response_text = response_text.replace('\\n', '\n')
response_text = response_text.replace('\\r', '')
response_text = response_text.replace('\\t', '\t')
response_text = response_text.replace('\\"', '"')
response_text = response_text.replace("\\'", "'")
response_text = response_text.replace('\\\\', '\\')
# Clean up markdown code blocks if present
if '```' in response_text:
parts = response_text.split('```')
for part in parts:
if 'csv' in part[:10].lower():
response_text = part[part.find('\n')+1:]
break
elif part.strip() and ('type,raw_name' in part or 'character,' in part or 'term,' in part):
response_text = part
break
# Normalize line endings
response_text = response_text.replace('\r\n', '\n').replace('\r', '\n')
lines = [line.strip() for line in response_text.strip().split('\n') if line.strip()]
csv_lines = []
header_found = False
# Check if we should skip frequency check
skip_frequency_check = os.getenv("GLOSSARY_SKIP_FREQUENCY_CHECK", "0") == "1"
# Add option to completely skip ALL validation for maximum speed
skip_all_validation = os.getenv("GLOSSARY_SKIP_ALL_VALIDATION", "0") == "1"
if skip_all_validation:
print("📑 ⚡ FAST MODE: Skipping all frequency validation (accepting all AI results)")
# Always use the enforced 3-column header
csv_lines.append("type,raw_name,translated_name")
# Process the AI response
for line in lines:
# Skip header lines
if 'type' in line.lower() and 'raw_name' in line.lower():
continue
# Parse CSV line
parts = [p.strip().strip('"\"') for p in line.split(',')]
if len(parts) >= 3:
# Has all 3 columns
entry_type = parts[0]
raw_name = parts[1]
translated_name = parts[2]
if raw_name and translated_name:
csv_lines.append(f"{entry_type},{raw_name},{translated_name}")
elif len(parts) == 2:
# Missing type, default to 'term'
raw_name = parts[0]
translated_name = parts[1]
if raw_name and translated_name:
csv_lines.append(f"term,{raw_name},{translated_name}")
print(f"📑 Fast mode: Accepted {len(csv_lines) - 1} entries without validation")
return csv_lines
# For "only_with_honorifics" mode, ALWAYS skip frequency check
if filter_mode == "only_with_honorifics":
skip_frequency_check = True
print("📑 Filter mode 'only_with_honorifics': Bypassing frequency checks")
print(f"📑 Processing {len(lines)} lines from AI response...")
print(f"📑 Text corpus size: {len(all_text):,} chars")
print(f"📑 Frequency checking: {'DISABLED' if skip_frequency_check else f'ENABLED (min: {min_frequency})'}")
print(f"📑 Fuzzy threshold: {fuzzy_threshold}")
# Collect all terms first for batch processing
all_terms_to_check = []
term_info_map = {} # Map term to its full info
if not skip_frequency_check:
# First pass: collect all terms that need frequency checking
for line in lines:
if 'type' in line.lower() and 'raw_name' in line.lower():
continue # Skip header
parts = [p.strip().strip('"\"') for p in line.split(',')]
if len(parts) >= 3:
entry_type = parts[0].lower()
raw_name = parts[1]
translated_name = parts[2]
elif len(parts) == 2:
entry_type = 'term'
raw_name = parts[0]
translated_name = parts[1]
else:
continue
if raw_name and translated_name:
# Store for batch processing
original_raw = raw_name
if strip_honorifics:
raw_name = self._strip_honorific(raw_name, language)
all_terms_to_check.append(raw_name)
term_info_map[raw_name] = {
'entry_type': entry_type,
'original_raw': original_raw,
'translated_name': translated_name,
'line': line
}
# Batch compute all frequencies at once
if all_terms_to_check:
print(f"📑 Computing frequencies for {len(all_terms_to_check)} terms...")
term_frequencies = self._batch_compute_frequencies(
all_terms_to_check, all_text, fuzzy_threshold, min_frequency
)
else:
term_frequencies = {}
# Now process the results using pre-computed frequencies
entries_processed = 0
entries_accepted = 0
# Process based on mode
if filter_mode == "only_with_honorifics" or skip_frequency_check:
# For these modes, accept all entries
csv_lines.append("type,raw_name,translated_name") # Header
for line in lines:
if 'type' in line.lower() and 'raw_name' in line.lower():
continue # Skip header
parts = [p.strip().strip('"\"') for p in line.split(',')]
if len(parts) >= 3:
entry_type = parts[0].lower()
raw_name = parts[1]
translated_name = parts[2]
elif len(parts) == 2:
entry_type = 'term'
raw_name = parts[0]
translated_name = parts[1]
else:
continue
if raw_name and translated_name:
csv_line = f"{entry_type},{raw_name},{translated_name}"
csv_lines.append(csv_line)
entries_accepted += 1
print(f"📑 Accepted {entries_accepted} entries (frequency check disabled)")
else:
# Use pre-computed frequencies
csv_lines.append("type,raw_name,translated_name") # Header
for term, info in term_info_map.items():
count = term_frequencies.get(term, 0)
# Also check original form if it was stripped
if info['original_raw'] != term:
count += term_frequencies.get(info['original_raw'], 0)
if count >= min_frequency:
csv_line = f"{info['entry_type']},{term},{info['translated_name']}"
csv_lines.append(csv_line)
entries_accepted += 1
# Log first few examples
if entries_accepted <= 5:
print(f"📑 ✓ Example: {term} -> {info['translated_name']} (freq: {count})")
print(f"📑 Frequency filtering complete: {entries_accepted}/{len(term_info_map)} terms accepted")
# Ensure we have at least the header
if len(csv_lines) == 0:
csv_lines.append("type,raw_name,translated_name")
# Print final summary
print(f"📑 Processing complete: {entries_accepted} terms accepted")
return csv_lines
def _deduplicate_glossary_with_fuzzy(self, csv_lines, fuzzy_threshold):
"""Apply fuzzy matching to remove duplicate entries from the glossary with stop flag checks"""
from difflib import SequenceMatcher
print(f"📑 Applying fuzzy deduplication (threshold: {fuzzy_threshold})...")
# Check stop flag at start
if is_stop_requested():
print(f"📑 ❌ Deduplication stopped by user")
return csv_lines
header_line = csv_lines[0] # Keep header
entry_lines = csv_lines[1:] # Data lines
deduplicated = [header_line]
seen_entries = {} # Use dict for O(1) lookups instead of list
seen_names_lower = set() # Quick exact match check
removed_count = 0
total_entries = len(entry_lines)
# Pre-process all entries for faster comparison
print(f"📑 Processing {total_entries} entries for deduplication...")
for idx, line in enumerate(entry_lines):
# Check stop flag every 100 entries
if idx > 0 and idx % 100 == 0:
if is_stop_requested():
print(f"📑 ❌ Deduplication stopped at entry {idx}/{total_entries}")
return deduplicated
# Show progress for large glossaries
if total_entries > 500 and idx % 200 == 0:
progress = (idx / total_entries) * 100
print(f"📑 Deduplication progress: {progress:.1f}% ({idx}/{total_entries})")
if not line.strip():
continue
parts = [p.strip() for p in line.split(',')]
if len(parts) < 3:
continue
entry_type = parts[0]
raw_name = parts[1]
translated_name = parts[2]
raw_name_lower = raw_name.lower()
# Fast exact duplicate check first
if raw_name_lower in seen_names_lower:
removed_count += 1
continue
# For fuzzy matching, only check if threshold is less than 1.0
is_duplicate = False
if fuzzy_threshold < 1.0:
# Use a more efficient approach: only check similar length strings
name_len = len(raw_name)
min_len = int(name_len * 0.7)
max_len = int(name_len * 1.3)
# Only compare with entries of similar length
candidates = []
for seen_name, (seen_type, seen_trans) in seen_entries.items():
if min_len <= len(seen_name) <= max_len:
candidates.append(seen_name)
# Check fuzzy similarity with candidates
for seen_name in candidates:
# Quick character overlap check before expensive SequenceMatcher
char_overlap = len(set(raw_name_lower) & set(seen_name.lower()))
if char_overlap < len(raw_name_lower) * 0.5:
continue # Too different, skip
raw_similarity = SequenceMatcher(None, raw_name_lower, seen_name.lower()).ratio()
if raw_similarity >= fuzzy_threshold:
if removed_count < 10: # Only log first few
print(f"📑 Removing duplicate: '{raw_name}' ~= '{seen_name}' (similarity: {raw_similarity:.2%})")
removed_count += 1
is_duplicate = True
break
if not is_duplicate:
seen_entries[raw_name] = (entry_type, translated_name)
seen_names_lower.add(raw_name_lower)
deduplicated.append(line)
print(f"📑 ✅ Removed {removed_count} duplicates from glossary")
print(f"📑 Final glossary size: {len(deduplicated) - 1} unique entries")
return deduplicated
def _merge_csv_entries(self, new_csv_lines, existing_glossary, strip_honorifics, language):
"""Merge CSV entries with existing glossary with stop flag checks"""
# Check stop flag at start
if is_stop_requested():
print(f"📑 ❌ Glossary merge stopped by user")
return new_csv_lines
# Parse existing glossary
existing_lines = []
existing_names = set()
if isinstance(existing_glossary, str):
# Already CSV format
lines = existing_glossary.strip().split('\n')
total_lines = len(lines)
for idx, line in enumerate(lines):
# Check stop flag every 50 lines
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Merge stopped while processing existing glossary at line {idx}/{total_lines}")
return new_csv_lines
if total_lines > 200:
progress = (idx / total_lines) * 100
print(f"📑 Processing existing glossary: {progress:.1f}%")
if 'type,raw_name' in line.lower():
continue # Skip header
line_stripped = line.strip()
# Skip token-efficient lines and section/bullet markers
if not line_stripped or line_stripped.startswith('===') or line_stripped.startswith('*') or line_stripped.lower().startswith('glossary:'):
continue
parts = [p.strip() for p in line.split(',')]
# Require at least 3 fields (type, raw_name, translated_name)
if len(parts) < 3:
continue
entry_type = parts[0].strip().lower()
# Only accept reasonable type tokens (letters/underscores only)
import re as _re
if not _re.match(r'^[a-z_]+$', entry_type):
continue
raw_name = parts[1]
if strip_honorifics:
raw_name = self._strip_honorific(raw_name, language)
parts[1] = raw_name
if raw_name not in existing_names:
existing_lines.append(','.join(parts))
existing_names.add(raw_name)
# Check stop flag before processing new names
if is_stop_requested():
print(f"📑 ❌ Merge stopped before processing new entries")
return new_csv_lines
# Get new names
new_names = set()
final_lines = []
for idx, line in enumerate(new_csv_lines):
# Check stop flag every 50 lines
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Merge stopped while processing new entries at line {idx}")
return final_lines if final_lines else new_csv_lines
if 'type,raw_name' in line.lower():
final_lines.append(line) # Keep header
continue
parts = [p.strip() for p in line.split(',')]
if len(parts) >= 2:
new_names.add(parts[1])
final_lines.append(line)
# Check stop flag before adding existing entries
if is_stop_requested():
print(f"📑 ❌ Merge stopped before combining entries")
return final_lines
# Add non-duplicate existing entries
added_count = 0
for idx, line in enumerate(existing_lines):
# Check stop flag every 50 additions
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Merge stopped while adding existing entries ({added_count} added)")
return final_lines
parts = [p.strip() for p in line.split(',')]
if len(parts) >= 2 and parts[1] not in new_names:
final_lines.append(line)
added_count += 1
print(f"📑 Merged {added_count} entries from existing glossary")
return final_lines
def _extract_with_patterns(self, all_text, language, min_frequency,
max_names, max_titles, batch_size,
existing_glossary, output_dir,
strip_honorifics=True, fuzzy_threshold=0.90, filter_mode='all'):
"""Extract glossary using pattern matching with true CSV format output and stop flag checks"""
print("📑 Using pattern-based extraction")
# Check stop flag at start
if is_stop_requested():
print("📑 ❌ Pattern-based extraction stopped by user")
return {}
def is_valid_name(name, language_hint='unknown'):
"""Strict validation for proper names only"""
if not name or len(name.strip()) < 1:
return False
name = name.strip()
if name.lower() in self.pattern_manager.COMMON_WORDS or name in self.pattern_manager.COMMON_WORDS:
return False
if language_hint == 'korean':
if not (2 <= len(name) <= 4):
return False
if not all(0xAC00 <= ord(char) <= 0xD7AF for char in name):
return False
if len(set(name)) == 1:
return False
elif language_hint == 'japanese':
if not (2 <= len(name) <= 6):
return False
has_kanji = any(0x4E00 <= ord(char) <= 0x9FFF for char in name)
has_kana = any((0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF) for char in name)
if not (has_kanji or has_kana):
return False
elif language_hint == 'chinese':
if not (2 <= len(name) <= 4):
return False
if not all(0x4E00 <= ord(char) <= 0x9FFF for char in name):
return False
elif language_hint == 'english':
if not name[0].isupper():
return False
if sum(1 for c in name if c.isalpha()) < len(name) * 0.8:
return False
if not (2 <= len(name) <= 20):
return False
return True
def detect_language_hint(text_sample):
"""Quick language detection for validation purposes"""
sample = text_sample[:1000]
korean_chars = sum(1 for char in sample if 0xAC00 <= ord(char) <= 0xD7AF)
japanese_kana = sum(1 for char in sample if (0x3040 <= ord(char) <= 0x309F) or (0x30A0 <= ord(char) <= 0x30FF))
chinese_chars = sum(1 for char in sample if 0x4E00 <= ord(char) <= 0x9FFF)
latin_chars = sum(1 for char in sample if 0x0041 <= ord(char) <= 0x007A)
if korean_chars > 50:
return 'korean'
elif japanese_kana > 20:
return 'japanese'
elif chinese_chars > 50 and japanese_kana < 10:
return 'chinese'
elif latin_chars > 100:
return 'english'
else:
return 'unknown'
language_hint = detect_language_hint(all_text)
print(f"📑 Detected primary language: {language_hint}")
# Check stop flag after language detection
if is_stop_requested():
print("📑 ❌ Extraction stopped after language detection")
return {}
honorifics_to_use = []
if language_hint in self.pattern_manager.CJK_HONORIFICS:
honorifics_to_use.extend(self.pattern_manager.CJK_HONORIFICS[language_hint])
honorifics_to_use.extend(self.pattern_manager.CJK_HONORIFICS.get('english', []))
print(f"📑 Using {len(honorifics_to_use)} honorifics for {language_hint}")
names_with_honorifics = {}
standalone_names = {}
# Check if parallel processing is enabled
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
# PARALLEL HONORIFIC PROCESSING
if extraction_workers > 1 and len(honorifics_to_use) > 3:
print(f"📑 Scanning for names with honorifics (parallel with {extraction_workers} workers)...")
# Create a wrapper function that can be called in parallel
def process_honorific(args):
"""Process a single honorific in a worker thread"""
honorific, idx, total = args
# Check stop flag
if is_stop_requested():
return None, None
print(f"📑 Worker processing honorific {idx}/{total}: '{honorific}'")
# Local dictionaries for this worker
local_names_with = {}
local_standalone = {}
# Call the extraction method
self._extract_names_for_honorific(
honorific, all_text, language_hint,
min_frequency, local_names_with,
local_standalone, is_valid_name, fuzzy_threshold
)
return local_names_with, local_standalone
# Prepare arguments for parallel processing
honorific_args = [
(honorific, idx + 1, len(honorifics_to_use))
for idx, honorific in enumerate(honorifics_to_use)
]
# Process honorifics in parallel
with ThreadPoolExecutor(max_workers=min(extraction_workers, len(honorifics_to_use))) as executor:
futures = []
for args in honorific_args:
if is_stop_requested():
executor.shutdown(wait=False)
return {}
future = executor.submit(process_honorific, args)
futures.append(future)
# Collect results as they complete
completed = 0
for future in as_completed(futures):
if is_stop_requested():
executor.shutdown(wait=False)
return {}
try:
result = future.result()
if result and result[0] is not None:
local_names_with, local_standalone = result
# Merge results (thread-safe since we're in main thread)
for name, count in local_names_with.items():
if name not in names_with_honorifics:
names_with_honorifics[name] = count
else:
names_with_honorifics[name] = max(names_with_honorifics[name], count)
for name, count in local_standalone.items():
if name not in standalone_names:
standalone_names[name] = count
else:
standalone_names[name] = max(standalone_names[name], count)
completed += 1
if completed % 5 == 0 or completed == len(honorifics_to_use):
print(f"📑 Honorific processing: {completed}/{len(honorifics_to_use)} completed")
except Exception as e:
print(f"⚠️ Failed to process honorific: {e}")
completed += 1
print(f"📑 Parallel honorific processing completed: found {len(names_with_honorifics)} names")
else:
# SEQUENTIAL PROCESSING (fallback)
print("📑 Scanning for names with honorifics...")
# Extract names with honorifics
total_honorifics = len(honorifics_to_use)
for idx, honorific in enumerate(honorifics_to_use):
# Check stop flag before each honorific
if is_stop_requested():
print(f"📑 ❌ Extraction stopped at honorific {idx}/{total_honorifics}")
return {}
print(f"📑 Processing honorific {idx + 1}/{total_honorifics}: '{honorific}'")
self._extract_names_for_honorific(honorific, all_text, language_hint,
min_frequency, names_with_honorifics,
standalone_names, is_valid_name, fuzzy_threshold)
# Check stop flag before processing terms
if is_stop_requested():
print("📑 ❌ Extraction stopped before processing terms")
return {}
# Apply filter mode
filtered_names = {}
if filter_mode == 'only_with_honorifics':
# Only keep names that have honorifics (no standalone names)
filtered_names = names_with_honorifics.copy()
print(f"📑 Filter: Keeping only names with honorifics ({len(filtered_names)} names)")
elif filter_mode == 'only_without_honorifics':
# Keep standalone names that were NOT found with honorifics
for name, count in standalone_names.items():
# Check if this name also appears with honorifics
appears_with_honorific = False
for honorific_name in names_with_honorifics.keys():
if self._strip_honorific(honorific_name, language_hint) == name:
appears_with_honorific = True
break
# Only add if it doesn't appear with honorifics
if not appears_with_honorific:
filtered_names[name] = count
print(f"📑 Filter: Keeping only names without honorifics ({len(filtered_names)} names)")
else: # 'all' mode
# Keep all names (both with and without honorifics)
filtered_names = names_with_honorifics.copy()
# Also add standalone names
for name, count in standalone_names.items():
if name not in filtered_names and not any(
self._strip_honorific(n, language_hint) == name for n in filtered_names.keys()
):
filtered_names[name] = count
print(f"📑 Filter: Keeping all names ({len(filtered_names)} names)")
# Process extracted terms
final_terms = {}
term_count = 0
total_terms = len(filtered_names)
for term, count in filtered_names.items():
term_count += 1
# Check stop flag every 20 terms
if term_count % 20 == 0:
if is_stop_requested():
print(f"📑 ❌ Term processing stopped at {term_count}/{total_terms}")
return {}
if strip_honorifics:
clean_term = self._strip_honorific(term, language_hint)
if clean_term in final_terms:
final_terms[clean_term] = final_terms[clean_term] + count
else:
final_terms[clean_term] = count
else:
final_terms[term] = count
# Check stop flag before finding titles
if is_stop_requested():
print("📑 ❌ Extraction stopped before finding titles")
return {}
# Find titles (but respect filter mode)
print("📑 Scanning for titles...")
found_titles = {}
# Extract titles for all modes EXCEPT "only_with_honorifics"
# (titles are included in "only_without_honorifics" since titles typically don't have honorifics)
if filter_mode != 'only_with_honorifics':
title_patterns_to_use = []
if language_hint in self.pattern_manager.TITLE_PATTERNS:
title_patterns_to_use.extend(self.pattern_manager.TITLE_PATTERNS[language_hint])
title_patterns_to_use.extend(self.pattern_manager.TITLE_PATTERNS.get('english', []))
total_patterns = len(title_patterns_to_use)
for pattern_idx, pattern in enumerate(title_patterns_to_use):
# Check stop flag before each pattern
if is_stop_requested():
print(f"📑 ❌ Title extraction stopped at pattern {pattern_idx}/{total_patterns}")
return {}
print(f"📑 Processing title pattern {pattern_idx + 1}/{total_patterns}")
matches = list(re.finditer(pattern, all_text, re.IGNORECASE if 'english' in pattern else 0))
for match_idx, match in enumerate(matches):
# Check stop flag every 50 matches
if match_idx > 0 and match_idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Title extraction stopped at match {match_idx}")
return {}
title = match.group(0)
# Skip if this title is already in names
if title in filtered_names or title in names_with_honorifics:
continue
count = self._find_fuzzy_matches(title, all_text, fuzzy_threshold)
# Check if stopped during fuzzy matching
if is_stop_requested():
print(f"📑 ❌ Title extraction stopped during fuzzy matching")
return {}
if count >= min_frequency:
if re.match(r'[A-Za-z]', title):
title = title.title()
if strip_honorifics:
title = self._strip_honorific(title, language_hint)
if title not in found_titles:
found_titles[title] = count
if filter_mode == 'only_without_honorifics':
print(f"📑 Found {len(found_titles)} titles (included in 'without honorifics' mode)")
else:
print(f"📑 Found {len(found_titles)} unique titles")
else:
print(f"📑 Skipping title extraction (filter mode: only_with_honorifics)")
# Check stop flag before sorting and translation
if is_stop_requested():
print("📑 ❌ Extraction stopped before sorting terms")
return {}
# Combine and sort
sorted_names = sorted(final_terms.items(), key=lambda x: x[1], reverse=True)[:max_names]
sorted_titles = sorted(found_titles.items(), key=lambda x: x[1], reverse=True)[:max_titles]
all_terms = []
for name, count in sorted_names:
all_terms.append(name)
for title, count in sorted_titles:
all_terms.append(title)
print(f"📑 Total terms to translate: {len(all_terms)}")
# Check stop flag before translation
if is_stop_requested():
print("📑 ❌ Extraction stopped before translation")
return {}
# Translate terms
if os.getenv("DISABLE_GLOSSARY_TRANSLATION", "0") == "1":
print("📑 Translation disabled - keeping original terms")
translations = {term: term for term in all_terms}
else:
print(f"📑 Translating {len(all_terms)} terms...")
translations = self._translate_terms_batch(all_terms, language_hint, batch_size, output_dir)
# Check if translation was stopped
if is_stop_requested():
print("📑 ❌ Extraction stopped after translation")
return translations # Return partial results
# Build CSV lines
csv_lines = ["type,raw_name,translated_name"]
for name, _ in sorted_names:
if name in translations:
csv_lines.append(f"character,{name},{translations[name]}")
for title, _ in sorted_titles:
if title in translations:
csv_lines.append(f"term,{title},{translations[title]}")
# Check stop flag before merging
if is_stop_requested():
print("📑 ❌ Extraction stopped before merging with existing glossary")
# Still save what we have
csv_content = '\n'.join(csv_lines)
glossary_path = os.path.join(output_dir, "glossary.json")
self._atomic_write_file(glossary_path, csv_content)
return self._parse_csv_to_dict(csv_content)
# Merge with existing glossary
if existing_glossary:
csv_lines = self._merge_csv_entries(csv_lines, existing_glossary, strip_honorifics, language_hint)
# Check stop flag before deduplication
if is_stop_requested():
print("📑 ❌ Extraction stopped before deduplication")
csv_content = '\n'.join(csv_lines)
glossary_path = os.path.join(output_dir, "glossary.json")
self._atomic_write_file(glossary_path, csv_content)
return self._parse_csv_to_dict(csv_content)
# Fuzzy matching deduplication
csv_lines = self._deduplicate_glossary_with_fuzzy(csv_lines, fuzzy_threshold)
# Create CSV content
csv_content = '\n'.join(csv_lines)
# Save glossary as CSV
glossary_path = os.path.join(output_dir, "glossary.csv")
self._atomic_write_file(glossary_path, csv_content)
print(f"\n📑 ✅ TARGETED GLOSSARY SAVED!")
print(f"📑 File: {glossary_path}")
print(f"📑 Total entries: {len(csv_lines) - 1}") # Exclude header
return self._parse_csv_to_dict(csv_content)
def _translate_terms_batch(self, term_list, profile_name, batch_size=50, output_dir=None):
"""Use fully configurable prompts for translation with interrupt support"""
if not term_list or os.getenv("DISABLE_GLOSSARY_TRANSLATION", "0") == "1":
print(f"📑 Glossary translation disabled or no terms to translate")
return {term: term for term in term_list}
# Check stop flag
if is_stop_requested():
print("📑 ❌ Glossary translation stopped by user")
return {term: term for term in term_list}
try:
MODEL = os.getenv("MODEL", "gemini-1.5-flash")
API_KEY = (os.getenv("API_KEY") or
os.getenv("OPENAI_API_KEY") or
os.getenv("OPENAI_OR_Gemini_API_KEY") or
os.getenv("GEMINI_API_KEY"))
if is_traditional_translation_api(MODEL):
return
if not API_KEY:
print(f"📑 No API key found, skipping translation")
return {term: term for term in term_list}
print(f"📑 Translating {len(term_list)} {profile_name} terms to English using batch size {batch_size}...")
from unified_api_client import UnifiedClient, UnifiedClientError
client = UnifiedClient(model=MODEL, api_key=API_KEY, output_dir=output_dir)
if hasattr(client, 'reset_cleanup_state'):
client.reset_cleanup_state()
# Get custom translation prompt from environment
translation_prompt_template = os.getenv("GLOSSARY_TRANSLATION_PROMPT", "")
if not translation_prompt_template:
translation_prompt_template = """You are translating {language} character names and important terms to English.
For character names, provide English transliterations or keep as romanized.
Keep honorifics/suffixes only if they are integral to the name.
Respond with the same numbered format.
Terms to translate:
{terms_list}
Provide translations in the same numbered format."""
all_translations = {}
chunk_timeout = int(os.getenv("CHUNK_TIMEOUT", "300")) # 5 minute default
for i in range(0, len(term_list), batch_size):
# Check stop flag before each batch
if is_stop_requested():
print(f"📑 ❌ Translation stopped at batch {(i // batch_size) + 1}")
# Return partial translations
for term in term_list:
if term not in all_translations:
all_translations[term] = term
return all_translations
batch = term_list[i:i + batch_size]
batch_num = (i // batch_size) + 1
total_batches = (len(term_list) + batch_size - 1) // batch_size
print(f"📑 Processing batch {batch_num}/{total_batches} ({len(batch)} terms)...")
# Format terms list
terms_text = ""
for idx, term in enumerate(batch, 1):
terms_text += f"{idx}. {term}\n"
# Replace placeholders in prompt
prompt = translation_prompt_template.replace('{language}', profile_name)
prompt = prompt.replace('{terms_list}', terms_text.strip())
prompt = prompt.replace('{batch_size}', str(len(batch)))
messages = [
{"role": "user", "content": prompt}
]
try:
temperature = float(os.getenv("TEMPERATURE", "0.3"))
max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "4096"))
# Use send_with_interrupt for interruptible API call
print(f"📑 Sending translation request for batch {batch_num} (interruptible)...")
response = send_with_interrupt(
messages=messages,
client=client,
temperature=temperature,
max_tokens=max_tokens,
stop_check_fn=is_stop_requested,
chunk_timeout=chunk_timeout
)
# Handle response properly
if hasattr(response, 'content'):
response_text = response.content
else:
response_text = str(response)
batch_translations = self._parse_translation_response(response_text, batch)
all_translations.update(batch_translations)
print(f"📑 Batch {batch_num} completed: {len(batch_translations)} translations")
# Small delay between batches to avoid rate limiting (configurable)
if i + batch_size < len(term_list):
# Check stop before sleep
if is_stop_requested():
print(f"📑 ❌ Translation stopped after batch {batch_num}")
# Fill in missing translations
for term in term_list:
if term not in all_translations:
all_translations[term] = term
return all_translations
# Use configurable batch delay or default to 0.1s (much faster than 0.5s)
batch_delay = float(os.getenv("GLOSSARY_BATCH_DELAY", "0.001"))
if batch_delay > 0:
time.sleep(batch_delay)
except UnifiedClientError as e:
if "stopped by user" in str(e).lower():
print(f"📑 ❌ Translation interrupted by user at batch {batch_num}")
# Fill in remaining terms with originals
for term in term_list:
if term not in all_translations:
all_translations[term] = term
return all_translations
else:
print(f"⚠️ Translation failed for batch {batch_num}: {e}")
for term in batch:
all_translations[term] = term
except Exception as e:
print(f"⚠️ Translation failed for batch {batch_num}: {e}")
for term in batch:
all_translations[term] = term
# Ensure all terms have translations
for term in term_list:
if term not in all_translations:
all_translations[term] = term
translated_count = sum(1 for term, translation in all_translations.items()
if translation != term and translation.strip())
print(f"📑 Successfully translated {translated_count}/{len(term_list)} terms")
return all_translations
except Exception as e:
print(f"⚠️ Glossary translation failed: {e}")
return {term: term for term in term_list}
def _extract_names_for_honorific(self, honorific, all_text, language_hint,
min_frequency, names_with_honorifics,
standalone_names, is_valid_name, fuzzy_threshold=0.90):
"""Extract names for a specific honorific with fuzzy matching and stop flag checks"""
# Check stop flag at start
if is_stop_requested():
print(f"📑 ❌ Name extraction for '{honorific}' stopped by user")
return
if language_hint == 'korean' and not honorific.startswith('-'):
pattern = r'([\uac00-\ud7af]{2,4})(?=' + re.escape(honorific) + r'(?:\s|[,.\!?]|$))'
matches = list(re.finditer(pattern, all_text))
total_matches = len(matches)
for idx, match in enumerate(matches):
# Check stop flag every 50 matches
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Korean name extraction stopped at {idx}/{total_matches}")
return
# Show progress for large sets
if total_matches > 500:
progress = (idx / total_matches) * 100
print(f"📑 Processing Korean names: {progress:.1f}% ({idx}/{total_matches})")
potential_name = match.group(1)
if is_valid_name(potential_name, 'korean'):
full_form = potential_name + honorific
# Use fuzzy matching for counting with stop check
count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
# Check if stopped during fuzzy matching
if is_stop_requested():
print(f"📑 ❌ Name extraction stopped during fuzzy matching")
return
if count >= min_frequency:
context_patterns = [
full_form + r'[은는이가]',
full_form + r'[을를]',
full_form + r'[에게한테]',
r'["]' + full_form,
full_form + r'[,]',
]
context_count = 0
for ctx_pattern in context_patterns:
context_count += len(re.findall(ctx_pattern, all_text))
if context_count > 0:
names_with_honorifics[full_form] = count
standalone_names[potential_name] = count
elif language_hint == 'japanese' and not honorific.startswith('-'):
pattern = r'([\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]{2,5})(?=' + re.escape(honorific) + r'(?:\s|[、。！？]|$))'
matches = list(re.finditer(pattern, all_text))
total_matches = len(matches)
for idx, match in enumerate(matches):
# Check stop flag every 50 matches
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Japanese name extraction stopped at {idx}/{total_matches}")
return
if total_matches > 500:
progress = (idx / total_matches) * 100
print(f"📑 Processing Japanese names: {progress:.1f}% ({idx}/{total_matches})")
potential_name = match.group(1)
if is_valid_name(potential_name, 'japanese'):
full_form = potential_name + honorific
count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
if is_stop_requested():
print(f"📑 ❌ Name extraction stopped during fuzzy matching")
return
if count >= min_frequency:
names_with_honorifics[full_form] = count
standalone_names[potential_name] = count
elif language_hint == 'chinese' and not honorific.startswith('-'):
pattern = r'([\u4e00-\u9fff]{2,4})(?=' + re.escape(honorific) + r'(?:\s|[，。！？]|$))'
matches = list(re.finditer(pattern, all_text))
total_matches = len(matches)
for idx, match in enumerate(matches):
# Check stop flag every 50 matches
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ Chinese name extraction stopped at {idx}/{total_matches}")
return
if total_matches > 500:
progress = (idx / total_matches) * 100
print(f"📑 Processing Chinese names: {progress:.1f}% ({idx}/{total_matches})")
potential_name = match.group(1)
if is_valid_name(potential_name, 'chinese'):
full_form = potential_name + honorific
count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
if is_stop_requested():
print(f"📑 ❌ Name extraction stopped during fuzzy matching")
return
if count >= min_frequency:
names_with_honorifics[full_form] = count
standalone_names[potential_name] = count
elif honorific.startswith('-') or honorific.startswith(' '):
is_space_separated = honorific.startswith(' ')
if is_space_separated:
pattern_english = r'\b([A-Z][a-zA-Z]+)' + re.escape(honorific) + r'(?=\s|[,.\!?]|$)'
else:
pattern_english = r'\b([A-Z][a-zA-Z]+)' + re.escape(honorific) + r'\b'
matches = list(re.finditer(pattern_english, all_text))
total_matches = len(matches)
for idx, match in enumerate(matches):
# Check stop flag every 50 matches
if idx > 0 and idx % 50 == 0:
if is_stop_requested():
print(f"📑 ❌ English name extraction stopped at {idx}/{total_matches}")
return
if total_matches > 500:
progress = (idx / total_matches) * 100
print(f"📑 Processing English names: {progress:.1f}% ({idx}/{total_matches})")
potential_name = match.group(1)
if is_valid_name(potential_name, 'english'):
full_form = potential_name + honorific
count = self._find_fuzzy_matches(full_form, all_text, fuzzy_threshold)
if is_stop_requested():
print(f"📑 ❌ Name extraction stopped during fuzzy matching")
return
if count >= min_frequency:
names_with_honorifics[full_form] = count
standalone_names[potential_name] = count
def _parse_translation_response(self, response, original_terms):
"""Parse translation response - handles numbered format"""
translations = {}
# Handle UnifiedResponse object
if hasattr(response, 'content'):
response_text = response.content
else:
response_text = str(response)
lines = response_text.strip().split('\n')
for line in lines:
line = line.strip()
if not line or not line[0].isdigit():
continue
try:
number_match = re.match(r'^(\d+)\.?\s*(.+)', line)
if number_match:
num = int(number_match.group(1)) - 1
content = number_match.group(2).strip()
if 0 <= num < len(original_terms):
original_term = original_terms[num]
for separator in ['->', '→', ':', '-', '—', '=']:
if separator in content:
parts = content.split(separator, 1)
if len(parts) == 2:
translation = parts[1].strip()
translation = translation.strip('"\'()[]')
if translation and translation != original_term:
translations[original_term] = translation
break
else:
if content != original_term:
translations[original_term] = content
except (ValueError, IndexError):
continue
return translations
# =====================================================
# UNIFIED UTILITIES
# =====================================================
def sanitize_resource_filename(filename):
"""Sanitize resource filenames for filesystem compatibility"""
filename = unicodedata.normalize('NFC', filename)
replacements = {
'/': '_', '\\': '_', ':': '_', '*': '_',
'?': '_', '"': '_', '<': '_', '>': '_',
'|': '_', '\0': '', '\n': '_', '\r': '_'
}
for old, new in replacements.items():
filename = filename.replace(old, new)
filename = ''.join(char for char in filename if ord(char) >= 32)
name, ext = os.path.splitext(filename)
if not name:
name = 'resource'
return name + ext
def should_retain_source_extension():
"""Read GUI toggle for retaining original extension and no 'response_' prefix.
This is stored in config or env by the GUI; we read env as bridge.
"""
return os.getenv('RETAIN_SOURCE_EXTENSION', os.getenv('retain_source_extension', '0')) in ('1', 'true', 'True')
def make_safe_filename(title, actual_num):
"""Create a safe filename that works across different filesystems"""
if not title:
return f"chapter_{actual_num:03d}"
title = unicodedata.normalize('NFC', str(title))
dangerous_chars = {
'/': '_', '\\': '_', ':': '_', '*': '_', '?': '_',
'"': '_', '<': '_', '>': '_', '|': '_', '\0': '',
'\n': ' ', '\r': ' ', '\t': ' '
}
for old, new in dangerous_chars.items():
title = title.replace(old, new)
title = ''.join(char for char in title if ord(char) >= 32)
title = re.sub(r'\s+', '_', title)
title = title.strip('_.• \t')
if not title or title == '_' * len(title):
title = f"chapter_{actual_num:03d}"
return title
def get_content_hash(html_content):
"""Create a stable hash of content"""
return ContentProcessor.get_content_hash(html_content)
def clean_ai_artifacts(text, remove_artifacts=True):
"""Remove AI response artifacts from text"""
return ContentProcessor.clean_ai_artifacts(text, remove_artifacts)
def find_glossary_file(output_dir):
"""Return path to glossary file preferring CSV over JSON, or None if not found"""
candidates = [
os.path.join(output_dir, "glossary.csv"),
os.path.join(output_dir, "glossary.json"),
]
for p in candidates:
if os.path.exists(p):
return p
return None
def clean_memory_artifacts(text):
"""Remove any memory/summary artifacts"""
return ContentProcessor.clean_memory_artifacts(text)
def emergency_restore_paragraphs(text, original_html=None, verbose=True):
"""Emergency restoration when AI returns wall of text"""
return ContentProcessor.emergency_restore_paragraphs(text, original_html, verbose)
def is_meaningful_text_content(html_content):
"""Check if chapter has meaningful text beyond just structure"""
return ContentProcessor.is_meaningful_text_content(html_content)
# =====================================================
# GLOBAL SETTINGS AND FLAGS
# =====================================================
logging.basicConfig(level=logging.DEBUG)
try:
if hasattr(sys.stdout, 'reconfigure'):
sys.stdout.reconfigure(encoding='utf-8', errors='ignore')
except AttributeError:
if sys.stdout is None:
devnull = open(os.devnull, "wb")
sys.stdout = io.TextIOWrapper(devnull, encoding='utf-8', errors='ignore')
elif hasattr(sys.stdout, 'buffer'):
try:
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='ignore')
except:
pass
_stop_requested = False
def set_stop_flag(value):
"""Set the global stop flag"""
global _stop_requested
_stop_requested = value
def is_stop_requested():
"""Check if stop was requested"""
global _stop_requested
return _stop_requested
def set_output_redirect(log_callback=None):
"""Redirect print statements to a callback function for GUI integration"""
if log_callback:
class CallbackWriter:
def __init__(self, callback):
self.callback = callback
def write(self, text):
if text.strip():
self.callback(text.strip())
def flush(self):
pass
sys.stdout = CallbackWriter(log_callback)
# =====================================================
# EPUB AND FILE PROCESSING
# =====================================================
def extract_chapter_number_from_filename(filename, opf_spine_position=None, opf_spine_data=None):
"""Extract chapter number from filename, prioritizing OPF spine order"""
# Priority 1: Use OPF spine position if available
if opf_spine_position is not None:
# Handle special non-chapter files (always chapter 0)
filename_lower = filename.lower()
name_without_ext = os.path.splitext(filename)[0].lower()
# Check for special keywords OR no numbers present
special_keywords = ['title', 'toc', 'cover', 'index', 'copyright', 'preface', 'nav']
has_special_keyword = any(name in filename_lower for name in special_keywords)
has_no_numbers = not re.search(r'\d', name_without_ext)
if has_special_keyword or has_no_numbers:
return 0, 'opf_special_file'
# Use spine position for regular chapters (0, 1, 2, 3...)
return opf_spine_position, 'opf_spine_order'
# Priority 2: Check if this looks like a special file (even without OPF)
name_without_ext = os.path.splitext(filename)[0].lower()
special_keywords = ['title', 'toc', 'cover', 'index', 'copyright', 'preface']
has_special_keyword = any(name in name_without_ext for name in special_keywords)
has_no_numbers = not re.search(r'\d', name_without_ext)
if has_special_keyword or has_no_numbers:
return 0, 'special_file'
# Priority 3: Try to extract sequential numbers (000, 001, 002...)
name_without_ext = os.path.splitext(filename)[0]
# Look for simple sequential patterns first
# Priority 3: Try to extract sequential numbers and decimals
sequential_patterns = [
(r'^(\d+)\.(\d+)$', 'decimal_number'), # 1.5, 2.3 (NEW!)
(r'^(\d{3,4})$', 'sequential_number'), # 000, 001, 0001
(r'^(\d+)$', 'direct_number'), # 0, 1, 2
]
for pattern, method in sequential_patterns:
match = re.search(pattern, name_without_ext)
if match:
if method == 'decimal_number':
# Return as float for decimal chapters
return float(f"{match.group(1)}.{match.group(2)}"), method
else:
return int(match.group(1)), method
# Priority 4: Fall back to existing filename parsing patterns
fallback_patterns = [
(r'^response_(\d+)[_\.]', 'response_prefix'),
(r'[Cc]hapter[_\s]*(\d+)', 'chapter_word'),
(r'[Cc]h[_\s]*(\d+)', 'ch_abbreviation'),
(r'No(\d+)', 'no_prefix'),
(r'第(\d+)[章话回]', 'chinese_chapter'),
(r'-h-(\d+)', 'h_suffix'), # For your -h-16 pattern
(r'_(\d+)', 'underscore_suffix'),
(r'-(\d+)', 'dash_suffix'),
(r'(\d+)', 'trailing_number'),
]
for pattern, method in fallback_patterns:
match = re.search(pattern, name_without_ext, re.IGNORECASE)
if match:
return int(match.group(1)), method
return None, None
def process_chapter_images(chapter_html: str, actual_num: int, image_translator: ImageTranslator,
check_stop_fn=None) -> Tuple[str, Dict[str, str]]:
"""Process and translate images in a chapter"""
from bs4 import BeautifulSoup
images = image_translator.extract_images_from_chapter(chapter_html)
if not images:
return chapter_html, {}
print(f"🖼️ Found {len(images)} images in chapter {actual_num}")
soup = BeautifulSoup(chapter_html, 'html.parser')
image_translations = {}
translated_count = 0
max_images_per_chapter = int(os.getenv('MAX_IMAGES_PER_CHAPTER', '10'))
if len(images) > max_images_per_chapter:
print(f" ⚠️ Chapter has {len(images)} images - processing first {max_images_per_chapter} only")
images = images[:max_images_per_chapter]
for idx, img_info in enumerate(images, 1):
if check_stop_fn and check_stop_fn():
print("❌ Image translation stopped by user")
break
img_src = img_info['src']
if img_src.startswith('../'):
img_path = os.path.join(image_translator.output_dir, img_src[3:])
elif img_src.startswith('./'):
img_path = os.path.join(image_translator.output_dir, img_src[2:])
elif img_src.startswith('/'):
img_path = os.path.join(image_translator.output_dir, img_src[1:])
else:
possible_paths = [
os.path.join(image_translator.images_dir, os.path.basename(img_src)),
os.path.join(image_translator.output_dir, img_src),
os.path.join(image_translator.output_dir, 'images', os.path.basename(img_src)),
os.path.join(image_translator.output_dir, os.path.basename(img_src)),
os.path.join(image_translator.output_dir, os.path.dirname(img_src), os.path.basename(img_src))
]
img_path = None
for path in possible_paths:
if os.path.exists(path):
img_path = path
print(f" ✅ Found image at: {path}")
break
if not img_path:
print(f" ❌ Image not found in any location for: {img_src}")
print(f" Tried: {possible_paths}")
continue
img_path = os.path.normpath(img_path)
if not os.path.exists(img_path):
print(f" ⚠️ Image not found: {img_path}")
print(f" 📁 Images directory: {image_translator.images_dir}")
print(f" 📁 Output directory: {image_translator.output_dir}")
print(f" 📁 Working directory: {os.getcwd()}")
if os.path.exists(image_translator.images_dir):
files = os.listdir(image_translator.images_dir)
print(f" 📁 Files in images dir: {files[:5]}...")
continue
print(f" 🔍 Processing image {idx}/{len(images)}: {os.path.basename(img_path)}")
context = ""
if img_info.get('alt'):
context += f", Alt text: {img_info['alt']}"
if translated_count > 0:
delay = float(os.getenv('IMAGE_API_DELAY', '1.0'))
time.sleep(delay)
translation_result = image_translator.translate_image(img_path, context, check_stop_fn)
print(f"\n🔍 DEBUG: Image {idx}/{len(images)}")
print(f" Translation result: {'Success' if translation_result and '[Image Translation Error:' not in translation_result else 'Failed'}")
if translation_result and "[Image Translation Error:" in translation_result:
print(f" Error message: {translation_result}")
if translation_result:
img_tag = None
for img in soup.find_all('img'):
if img.get('src') == img_src:
img_tag = img
break
if img_tag:
hide_label = os.getenv("HIDE_IMAGE_TRANSLATION_LABEL", "0") == "1"
print(f" 🔍 DEBUG: Integration Phase")
print(f" 🏷️ Hide label mode: {hide_label}")
print(f" 📍 Found img tag: {img_tag.get('src')}")
# Store the translation result in the dictionary FIRST
image_translations[img_path] = translation_result
# Parse the translation result to integrate into the chapter HTML
if '' in translation_result:
trans_soup = BeautifulSoup(translation_result, 'html.parser')
# Try to get the full container first
full_container = trans_soup.find('div', class_=['translated-text-only', 'image-with-translation'])
if full_container:
# Clone the container to avoid issues
new_container = BeautifulSoup(str(full_container), 'html.parser').find('div')
img_tag.replace_with(new_container)
print(f" ✅ Replaced image with full translation container")
else:
# Fallback: manually build the structure
trans_div = trans_soup.find('div', class_='image-translation')
if trans_div:
container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
img_tag.replace_with(container)
if not hide_label:
new_img = soup.new_tag('img', src=img_src)
if img_info.get('alt'):
new_img['alt'] = img_info.get('alt')
container.append(new_img)
# Clone the translation div content
new_trans_div = soup.new_tag('div', **{'class': 'image-translation'})
# Copy all children from trans_div to new_trans_div
for child in trans_div.children:
if hasattr(child, 'name'):
new_trans_div.append(BeautifulSoup(str(child), 'html.parser'))
else:
new_trans_div.append(str(child))
container.append(new_trans_div)
print(f" ✅ Built container with translation div")
else:
print(f" ⚠️ No translation div found in result")
continue
else:
# Plain text translation - build structure manually
container = soup.new_tag('div', **{'class': 'translated-text-only' if hide_label else 'image-with-translation'})
img_tag.replace_with(container)
if not hide_label:
new_img = soup.new_tag('img', src=img_src)
if img_info.get('alt'):
new_img['alt'] = img_info.get('alt')
container.append(new_img)
# Create translation div with content
translation_div = soup.new_tag('div', **{'class': 'image-translation'})
if not hide_label:
label_p = soup.new_tag('p')
label_em = soup.new_tag('em')
#label_em.string = "[Image text translation:]"
label_p.append(label_em)
translation_div.append(label_p)
trans_p = soup.new_tag('p')
trans_p.string = translation_result
translation_div.append(trans_p)
container.append(translation_div)
print(f" ✅ Created plain text translation structure")
translated_count += 1
# Save to translated_images folder
trans_filename = f"ch{actual_num:03d}_img{idx:02d}_translation.html"
trans_filepath = os.path.join(image_translator.translated_images_dir, trans_filename)
# Extract just the translation content for saving
save_soup = BeautifulSoup(translation_result, 'html.parser')
save_div = save_soup.find('div', class_='image-translation')
if not save_div:
# Create a simple div for plain text
save_div = f'
'
with open(trans_filepath, 'w', encoding='utf-8') as f:
f.write(f"""
Chapter {actual_num} - Image {idx} Translation
Chapter {actual_num} - Image {idx}
Original: {os.path.basename(img_path)}
{save_div}
""")
print(f" ✅ Saved translation to: {trans_filename}")
else:
print(f" ⚠️ Could not find image tag in HTML for: {img_src}")
if translated_count > 0:
print(f" 🖼️ Successfully translated {translated_count} images")
# Debug output
final_html = str(soup)
trans_count = final_html.count('
')
print(f" 📊 Final HTML has {trans_count} translation divs")
print(f" 📊 image_translations dict has {len(image_translations)} entries")
prog = image_translator.load_progress()
if "image_chunks" in prog:
completed_images = []
for img_key, img_data in prog["image_chunks"].items():
if len(img_data["completed"]) == img_data["total"]:
completed_images.append(img_key)
for img_key in completed_images:
del prog["image_chunks"][img_key]
if completed_images:
image_translator.save_progress(prog)
print(f" 🧹 Cleaned up progress for {len(completed_images)} completed images")
image_translator.save_translation_log(actual_num, image_translations)
return str(soup), image_translations
else:
print(f" ℹ️ No images were successfully translated")
return chapter_html, {}
def detect_novel_numbering(chapters):
"""Detect if the novel uses 0-based or 1-based chapter numbering with improved accuracy"""
print("[DEBUG] Detecting novel numbering system...")
if not chapters:
return False
if isinstance(chapters[0], str):
print("[DEBUG] Text file detected, skipping numbering detection")
return False
patterns = PatternManager.FILENAME_EXTRACT_PATTERNS
# Special check for prefix_suffix pattern like "0000_1.xhtml"
prefix_suffix_pattern = r'^(\d+)_(\d+)[_\.]'
# Track chapter numbers from different sources
filename_numbers = []
content_numbers = []
has_prefix_suffix = False
prefix_suffix_numbers = []
for idx, chapter in enumerate(chapters):
extracted_num = None
# Check filename patterns
if 'original_basename' in chapter and chapter['original_basename']:
filename = chapter['original_basename']
elif 'filename' in chapter:
filename = os.path.basename(chapter['filename'])
else:
continue
# First check for prefix_suffix pattern
prefix_match = re.search(prefix_suffix_pattern, filename, re.IGNORECASE)
if prefix_match:
has_prefix_suffix = True
# Use the SECOND number (after underscore)
suffix_num = int(prefix_match.group(2))
prefix_suffix_numbers.append(suffix_num)
extracted_num = suffix_num
print(f"[DEBUG] Prefix_suffix pattern matched: {filename} -> Chapter {suffix_num}")
else:
# Try other patterns
for pattern in patterns:
match = re.search(pattern, filename)
if match:
extracted_num = int(match.group(1))
#print(f"[DEBUG] Pattern '{pattern}' matched: {filename} -> Chapter {extracted_num}")
break
if extracted_num is not None:
filename_numbers.append(extracted_num)
# Also check chapter content for chapter declarations
if 'body' in chapter:
# Look for "Chapter N" in the first 1000 characters
content_preview = chapter['body'][:1000]
content_match = re.search(r'Chapter\s+(\d+)', content_preview, re.IGNORECASE)
if content_match:
content_num = int(content_match.group(1))
content_numbers.append(content_num)
print(f"[DEBUG] Found 'Chapter {content_num}' in content")
# Decision logic with improved heuristics
# 1. If using prefix_suffix pattern, trust those numbers exclusively
if has_prefix_suffix and prefix_suffix_numbers:
min_suffix = min(prefix_suffix_numbers)
if min_suffix >= 1:
print(f"[DEBUG] ✅ 1-based novel detected (prefix_suffix pattern starts at {min_suffix})")
return False
else:
print(f"[DEBUG] ✅ 0-based novel detected (prefix_suffix pattern starts at {min_suffix})")
return True
# 2. If we have content numbers, prefer those over filename numbers
if content_numbers:
min_content = min(content_numbers)
# Check if we have a good sequence starting from 0 or 1
if 0 in content_numbers and 1 in content_numbers:
print(f"[DEBUG] ✅ 0-based novel detected (found both Chapter 0 and Chapter 1 in content)")
return True
elif min_content == 1:
print(f"[DEBUG] ✅ 1-based novel detected (content chapters start at 1)")
return False
# 3. Fall back to filename numbers
if filename_numbers:
min_filename = min(filename_numbers)
max_filename = max(filename_numbers)
# Check for a proper sequence
# If we have 0,1,2,3... it's likely 0-based
# If we have 1,2,3,4... it's likely 1-based
# Count how many chapters we have in sequence starting from 0
zero_sequence_count = 0
for i in range(len(chapters)):
if i in filename_numbers:
zero_sequence_count += 1
else:
break
# Count how many chapters we have in sequence starting from 1
one_sequence_count = 0
for i in range(1, len(chapters) + 1):
if i in filename_numbers:
one_sequence_count += 1
else:
break
print(f"[DEBUG] Zero-based sequence length: {zero_sequence_count}")
print(f"[DEBUG] One-based sequence length: {one_sequence_count}")
# If we have a better sequence starting from 1, it's 1-based
if one_sequence_count > zero_sequence_count and min_filename >= 1:
print(f"[DEBUG] ✅ 1-based novel detected (better sequence match starting from 1)")
return False
# If we have any 0 in filenames and it's part of a sequence
if 0 in filename_numbers and zero_sequence_count >= 3:
print(f"[DEBUG] ✅ 0-based novel detected (found 0 in sequence)")
return True
# 4. Default to 1-based if uncertain
print(f"[DEBUG] ✅ Defaulting to 1-based novel (insufficient evidence for 0-based)")
return False
def validate_chapter_continuity(chapters):
"""Validate chapter continuity and warn about issues"""
if not chapters:
print("No chapters to translate")
return
issues = []
# Get all chapter numbers
chapter_nums = [c['num'] for c in chapters]
actual_nums = [c.get('actual_chapter_num', c['num']) for c in chapters]
# Check for duplicates
duplicates = [num for num in chapter_nums if chapter_nums.count(num) > 1]
if duplicates:
issues.append(f"Duplicate chapter numbers found: {set(duplicates)}")
# Check for gaps in sequence
min_num = min(chapter_nums)
max_num = max(chapter_nums)
expected = set(range(min_num, max_num + 1))
actual = set(chapter_nums)
missing = expected - actual
if missing:
issues.append(f"Missing chapter numbers: {sorted(missing)}")
# Show gaps more clearly
gaps = []
sorted_missing = sorted(missing)
if sorted_missing:
start = sorted_missing[0]
end = sorted_missing[0]
for num in sorted_missing[1:]:
if num == end + 1:
end = num
else:
gaps.append(f"{start}-{end}" if start != end else str(start))
start = end = num
gaps.append(f"{start}-{end}" if start != end else str(start))
issues.append(f"Gap ranges: {', '.join(gaps)}")
# Check for duplicate titles
title_map = {}
for c in chapters:
title_lower = c['title'].lower().strip()
if title_lower in title_map:
title_map[title_lower].append(c['num'])
else:
title_map[title_lower] = [c['num']]
for title, nums in title_map.items():
if len(nums) > 1:
issues.append(f"Duplicate title '{title}' in chapters: {nums}")
# Print summary
print("\n" + "="*60)
print("📚 CHAPTER VALIDATION SUMMARY")
print("="*60)
print(f"Total chapters: {len(chapters)}")
print(f"Chapter range: {min_num} to {max_num}")
print(f"Expected count: {max_num - min_num + 1}")
print(f"Actual count: {len(chapters)}")
if len(chapters) != (max_num - min_num + 1):
print(f"⚠️ Chapter count mismatch - missing {(max_num - min_num + 1) - len(chapters)} chapters")
if issues:
print("\n⚠️ Issues found:")
for issue in issues:
print(f" - {issue}")
else:
print("✅ No continuity issues detected")
print("="*60 + "\n")
def validate_epub_structure(output_dir):
"""Validate that all necessary EPUB structure files are present"""
print("🔍 Validating EPUB structure...")
required_files = {
'container.xml': 'META-INF container file (critical)',
'*.opf': 'OPF package file (critical)',
'*.ncx': 'Navigation file (recommended)'
}
found_files = {}
missing_files = []
container_path = os.path.join(output_dir, 'container.xml')
if os.path.exists(container_path):
found_files['container.xml'] = 'Found'
print(" ✅ container.xml - Found")
else:
missing_files.append('container.xml')
print(" ❌ container.xml - Missing (CRITICAL)")
opf_files = []
ncx_files = []
for file in os.listdir(output_dir):
if file.lower().endswith('.opf'):
opf_files.append(file)
elif file.lower().endswith('.ncx'):
ncx_files.append(file)
if opf_files:
found_files['opf'] = opf_files
print(f" ✅ OPF file(s) - Found: {', '.join(opf_files)}")
else:
missing_files.append('*.opf')
print(" ❌ OPF file - Missing (CRITICAL)")
if ncx_files:
found_files['ncx'] = ncx_files
print(f" ✅ NCX file(s) - Found: {', '.join(ncx_files)}")
else:
missing_files.append('*.ncx')
print(" ⚠️ NCX file - Missing (navigation may not work)")
html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
if html_files:
print(f" ✅ Translated chapters - Found: {len(html_files)} files")
else:
print(" ⚠️ No translated chapter files found")
critical_missing = [f for f in missing_files if f in ['container.xml', '*.opf']]
if not critical_missing:
print("✅ EPUB structure validation PASSED")
print(" All critical files present for EPUB reconstruction")
return True
else:
print("❌ EPUB structure validation FAILED")
print(f" Missing critical files: {', '.join(critical_missing)}")
print(" EPUB reconstruction may fail without these files")
return False
def check_epub_readiness(output_dir):
"""Check if the output directory is ready for EPUB compilation"""
print("📋 Checking EPUB compilation readiness...")
issues = []
if not validate_epub_structure(output_dir):
issues.append("Missing critical EPUB structure files")
html_files = [f for f in os.listdir(output_dir) if f.lower().endswith('.html') and f.startswith('response_')]
if not html_files:
issues.append("No translated chapter files found")
else:
print(f" ✅ Found {len(html_files)} translated chapters")
metadata_path = os.path.join(output_dir, 'metadata.json')
if os.path.exists(metadata_path):
print(" ✅ Metadata file present")
try:
with open(metadata_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
if 'title' not in metadata:
issues.append("Metadata missing title")
except Exception as e:
issues.append(f"Metadata file corrupted: {e}")
else:
issues.append("Missing metadata.json file")
resource_dirs = ['css', 'fonts', 'images']
found_resources = 0
for res_dir in resource_dirs:
res_path = os.path.join(output_dir, res_dir)
if os.path.exists(res_path):
files = [f for f in os.listdir(res_path) if os.path.isfile(os.path.join(res_path, f))]
if files:
found_resources += len(files)
print(f" ✅ Found {len(files)} {res_dir} files")
if found_resources > 0:
print(f" ✅ Total resources: {found_resources} files")
else:
print(" ⚠️ No resource files found (this may be normal)")
if not issues:
print("🎉 EPUB compilation readiness: READY")
print(" All necessary files present for EPUB creation")
return True
else:
print("⚠️ EPUB compilation readiness: ISSUES FOUND")
for issue in issues:
print(f" • {issue}")
return False
def cleanup_previous_extraction(output_dir):
"""Clean up any files from previous extraction runs (preserves CSS files)"""
# Remove 'css' from cleanup_items to preserve CSS files
cleanup_items = [
'images', # Removed 'css' from this list
'.resources_extracted'
]
epub_structure_files = [
'container.xml', 'content.opf', 'toc.ncx'
]
cleaned_count = 0
# Clean up directories (except CSS)
for item in cleanup_items:
if item.startswith('.'):
continue
item_path = os.path.join(output_dir, item)
try:
if os.path.isdir(item_path):
shutil.rmtree(item_path)
print(f"🧹 Removed directory: {item}")
cleaned_count += 1
except Exception as e:
print(f"⚠️ Could not remove directory {item}: {e}")
# Clean up EPUB structure files
for epub_file in epub_structure_files:
file_path = os.path.join(output_dir, epub_file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
print(f"🧹 Removed EPUB file: {epub_file}")
cleaned_count += 1
except Exception as e:
print(f"⚠️ Could not remove {epub_file}: {e}")
# Clean up any loose .opf and .ncx files
try:
for file in os.listdir(output_dir):
if file.lower().endswith(('.opf', '.ncx')):
file_path = os.path.join(output_dir, file)
if os.path.isfile(file_path):
os.remove(file_path)
print(f"🧹 Removed EPUB file: {file}")
cleaned_count += 1
except Exception as e:
print(f"⚠️ Error scanning for EPUB files: {e}")
# Remove extraction marker
marker_path = os.path.join(output_dir, '.resources_extracted')
try:
if os.path.isfile(marker_path):
os.remove(marker_path)
print(f"🧹 Removed extraction marker")
cleaned_count += 1
except Exception as e:
print(f"⚠️ Could not remove extraction marker: {e}")
# Check if CSS files exist and inform user they're being preserved
css_path = os.path.join(output_dir, 'css')
if os.path.exists(css_path):
try:
css_files = [f for f in os.listdir(css_path) if os.path.isfile(os.path.join(css_path, f))]
if css_files:
print(f"📚 Preserving {len(css_files)} CSS files")
except Exception:
pass
if cleaned_count > 0:
print(f"🧹 Cleaned up {cleaned_count} items from previous runs (CSS files preserved)")
return cleaned_count
# =====================================================
# API AND TRANSLATION UTILITIES
# =====================================================
def send_with_interrupt(messages, client, temperature, max_tokens, stop_check_fn, chunk_timeout=None, request_id=None, context=None):
"""Send API request with interrupt capability and optional timeout retry.
Optional context parameter is passed through to the client to improve payload labeling.
"""
# Import UnifiedClientError at function level to avoid scoping issues
from unified_api_client import UnifiedClientError
# The client.send() call will handle multi-key rotation automatically
# Generate request_id if not provided
#if request_id is None:
# request_id = str(uuid.uuid4())[:8]
result_queue = queue.Queue()
def api_call():
try:
start_time = time.time()
# Check if client.send accepts request_id parameter
send_params = {
'messages': messages,
'temperature': temperature,
'max_tokens': max_tokens
}
# Add context if supported
sig = inspect.signature(client.send)
if 'context' in sig.parameters and context is not None:
send_params['context'] = context
# Add request_id if the client supports it
sig = inspect.signature(client.send)
#if 'request_id' in sig.parameters:
# send_params['request_id'] = request_id
result = client.send(**send_params)
elapsed = time.time() - start_time
result_queue.put((result, elapsed))
except Exception as e:
result_queue.put(e)
api_thread = threading.Thread(target=api_call)
api_thread.daemon = True
api_thread.start()
timeout = chunk_timeout if chunk_timeout is not None else 86400
check_interval = 0.5
elapsed = 0
while elapsed < timeout:
try:
result = result_queue.get(timeout=check_interval)
if isinstance(result, Exception):
# For expected errors like rate limits, preserve the error type without extra traceback
if hasattr(result, 'error_type') and result.error_type == "rate_limit":
raise result
elif "429" in str(result) or "rate limit" in str(result).lower():
# Convert generic exceptions to UnifiedClientError for rate limits
raise UnifiedClientError(str(result), error_type="rate_limit")
else:
raise result
if isinstance(result, tuple):
api_result, api_time = result
if chunk_timeout and api_time > chunk_timeout:
# Set cleanup flag when chunk timeout occurs
if hasattr(client, '_in_cleanup'):
client._in_cleanup = True
if hasattr(client, 'cancel_current_operation'):
client.cancel_current_operation()
raise UnifiedClientError(f"API call took {api_time:.1f}s (timeout: {chunk_timeout}s)")
return api_result
return result
except queue.Empty:
if stop_check_fn():
# Set cleanup flag when user stops
if hasattr(client, '_in_cleanup'):
client._in_cleanup = True
if hasattr(client, 'cancel_current_operation'):
client.cancel_current_operation()
raise UnifiedClientError("Translation stopped by user")
elapsed += check_interval
# Set cleanup flag when timeout occurs
if hasattr(client, '_in_cleanup'):
client._in_cleanup = True
if hasattr(client, 'cancel_current_operation'):
client.cancel_current_operation()
raise UnifiedClientError(f"API call timed out after {timeout} seconds")
def handle_api_error(processor, error, chunk_info=""):
"""Handle API errors with multi-key support"""
error_str = str(error)
# Check for rate limit
if "429" in error_str or "rate limit" in error_str.lower():
if processor.config.use_multi_api_keys:
print(f"⚠️ Rate limit hit {chunk_info}, client should rotate to next key")
stats = processor.client.get_stats()
print(f"📊 API Stats - Active keys: {stats.get('active_keys', 0)}/{stats.get('total_keys', 0)}")
if stats.get('active_keys', 0) == 0:
print("⏳ All API keys are cooling down - will wait and retry")
print(f"🔄 Multi-key error handling: Rate limit processed, preparing for key rotation...")
time.sleep(0.1) # Brief pause after rate limit detection for stability
return True # Always retry
else:
print(f"⚠️ Rate limit hit {chunk_info}, waiting before retry...")
time.sleep(60)
print(f"🔄 Single-key error handling: Rate limit wait completed, ready for retry...")
time.sleep(0.1) # Brief pause after rate limit wait for stability
return True # Always retry
# Other errors
print(f"❌ API Error {chunk_info}: {error_str}")
return False
def parse_token_limit(env_value):
"""Parse token limit from environment variable"""
if not env_value or env_value.strip() == "":
return None, "unlimited"
env_value = env_value.strip()
if env_value.lower() == "unlimited":
return None, "unlimited"
if env_value.isdigit() and int(env_value) > 0:
limit = int(env_value)
return limit, str(limit)
return 1000000, "1000000 (default)"
def build_system_prompt(user_prompt, glossary_path=None):
"""Build the system prompt with glossary - TRUE BRUTE FORCE VERSION"""
append_glossary = os.getenv("APPEND_GLOSSARY", "1") == "1"
actual_glossary_path = glossary_path
system = user_prompt if user_prompt else ""
if append_glossary and actual_glossary_path and os.path.exists(actual_glossary_path):
try:
print(f"[DEBUG] ✅ Loading glossary from: {os.path.abspath(actual_glossary_path)}")
# Try to load as JSON first
try:
with open(actual_glossary_path, "r", encoding="utf-8") as gf:
glossary_data = json.load(gf)
glossary_text = json.dumps(glossary_data, ensure_ascii=False, indent=2)
print(f"[DEBUG] Loaded as JSON")
except json.JSONDecodeError:
# If JSON fails, just read as raw text
#print(f"[DEBUG] JSON parse failed, reading as raw text")
with open(actual_glossary_path, "r", encoding="utf-8") as gf:
glossary_text = gf.read()
if system:
system += "\n\n"
custom_prompt = os.getenv("APPEND_GLOSSARY_PROMPT", "Character/Term Glossary (use these translations consistently):").strip()
if not custom_prompt:
custom_prompt = "Character/Term Glossary (use these translations consistently):"
system += f"{custom_prompt}\n{glossary_text}"
print(f"[DEBUG] ✅ Entire glossary appended!")
print(f"[DEBUG] Glossary text length: {len(glossary_text)} characters")
except Exception as e:
print(f"[ERROR] Could not load glossary: {e}")
import traceback
print(f"[ERROR] Full traceback: {traceback.format_exc()}")
else:
if not append_glossary:
#print(f"[DEBUG] ❌ Glossary append disabled")
pass
elif not actual_glossary_path:
print(f"[DEBUG] ❌ No glossary path provided")
elif not os.path.exists(actual_glossary_path):
print(f"[DEBUG] ❌ Glossary file does not exist: {actual_glossary_path}")
print(f"🎯 Final system prompt length: {len(system)} characters")
return system
def translate_title(title, client, system_prompt, user_prompt, temperature=0.3):
"""Translate the book title using the configured settings"""
if not title or not title.strip():
return title
print(f"📚 Processing book title: {title}")
try:
if os.getenv("TRANSLATE_BOOK_TITLE", "1") == "0":
print(f"📚 Book title translation disabled - keeping original")
return title
# Check if we're using a translation service (not AI)
client_type = getattr(client, 'client_type', '')
is_translation_service = client_type in ['deepl', 'google_translate']
if is_translation_service:
# For translation services, send only the text without AI prompts
print(f"📚 Using translation service ({client_type}) - sending text directly")
messages = [
{"role": "user", "content": title}
]
max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
else:
# For AI services, use prompts as before
book_title_prompt = os.getenv("BOOK_TITLE_PROMPT",
"Translate this book title to English while retaining any acronyms:")
# Get the system prompt for book titles, with fallback to default
book_title_system_prompt = os.getenv("BOOK_TITLE_SYSTEM_PROMPT",
"You are a translator. Respond with only the translated text, nothing else. Do not add any explanation or additional content.")
messages = [
{"role": "system", "content": book_title_system_prompt},
{"role": "user", "content": f"{book_title_prompt}\n\n{title}"}
]
max_tokens = int(os.getenv("MAX_OUTPUT_TOKENS", "8192"))
translated_title, _ = client.send(messages, temperature=temperature, max_tokens=max_tokens)
print(f"[DEBUG] Raw API response: '{translated_title}'")
print(f"[DEBUG] Response length: {len(translated_title)} (original: {len(title)})")
newline = '\n'
print(f"[DEBUG] Has newlines: {repr(translated_title) if newline in translated_title else 'No'}")
translated_title = translated_title.strip()
if ((translated_title.startswith('"') and translated_title.endswith('"')) or
(translated_title.startswith("'") and translated_title.endswith("'"))):
translated_title = translated_title[1:-1].strip()
if '\n' in translated_title:
print(f"⚠️ API returned multi-line content, keeping original title")
return title
# Check for JSON-like structured content, but allow simple brackets like [END]
if (any(char in translated_title for char in ['{', '}']) or
'"role":' in translated_title or
'"content":' in translated_title or
('[[' in translated_title and ']]' in translated_title)): # Only flag double brackets
print(f"⚠️ API returned structured content, keeping original title")
return title
if any(tag in translated_title.lower() for tag in ['
', '
', '
', '
', '
= 2:
return True
# Single strong error indicator in very short response
if len(content_str) < 50 and error_count >= 1:
return True
return False
# Additional helper function for debugging
def get_failure_reason(content):
"""
Returns the specific reason why content was marked as qa_failed
Useful for debugging and logging
"""
if not content:
return "Empty content"
content_str = str(content).strip()
content_lower = content_str.lower()
# Check each category and return the first match
failure_categories = {
"Explicit Failure Marker": [
"[TRANSLATION FAILED - ORIGINAL TEXT PRESERVED]",
"[IMAGE TRANSLATION FAILED]",
"API response unavailable",
"[]"
],
"HTTP Error": [
"authentication_error", "rate_limit_error", "api_error"
],
"Content Filter": [
"content_filter", "safety filter", "blocked by safety"
],
"Timeout": [
"timeout", "timed out", "apitimeouterror"
],
"Rate Limit": [
"rate limit exceeded", "quota exceeded", "too many requests"
],
"Refusal Pattern": [
"i cannot", "i can't", "unable to process"
],
"Empty Response": [
'"text": ""', "choices: [ { text: ''"
]
}
for category, markers in failure_categories.items():
for marker in markers:
if marker in content_str or marker in content_lower:
return f"{category}: {marker}"
if len(content_str) < 50:
return f"Short response with error indicators: {content_str[:30]}..."
return "Unknown failure pattern"
def convert_enhanced_text_to_html(plain_text, chapter_info=None):
"""Convert markdown/plain text back to HTML after translation (for enhanced mode)
This function handles the conversion of translated markdown back to HTML.
The input is the TRANSLATED text that was originally extracted using html2text.
"""
import re
preserve_structure = chapter_info.get('preserve_structure', False) if chapter_info else False
# First, try to use markdown2 for proper markdown conversion
try:
import markdown2
# Check if the text contains markdown patterns
has_markdown = any([
'##' in plain_text, # Headers
'**' in plain_text, # Bold
'*' in plain_text and not '**' in plain_text, # Italic
'[' in plain_text and '](' in plain_text, # Links
'```' in plain_text, # Code blocks
'> ' in plain_text, # Blockquotes
'- ' in plain_text or '* ' in plain_text or '1. ' in plain_text # Lists
])
if has_markdown or preserve_structure:
# Use markdown2 for proper conversion
html = markdown2.markdown(plain_text, extras=[
'cuddled-lists', # Lists without blank lines
'fenced-code-blocks', # Code blocks with ```
'break-on-newline', # Treat single newlines as
'smarty-pants', # Smart quotes and dashes
'tables', # Markdown tables
])
# Post-process to ensure proper paragraph structure
if not '' in html:
# If markdown2 didn't create paragraphs, wrap content
lines = html.split('\n')
processed_lines = []
for line in lines:
line = line.strip()
if line and not line.startswith('<') and not line.endswith('>'):
processed_lines.append(f'
{line}
')
elif line:
processed_lines.append(line)
html = '\n'.join(processed_lines)
return html
except ImportError:
print("⚠️ markdown2 not available, using fallback HTML conversion")
# Fallback: Manual markdown-to-HTML conversion
lines = plain_text.strip().split('\n')
html_parts = []
in_code_block = False
code_block_content = []
for line in lines:
# Handle code blocks
if line.strip().startswith('```'):
if in_code_block:
# End code block
html_parts.append('' + '\n'.join(code_block_content) + '
')
code_block_content = []
in_code_block = False
else:
# Start code block
in_code_block = True
continue
if in_code_block:
code_block_content.append(line)
continue
line = line.strip()
if not line:
# Preserve empty lines as paragraph breaks
if html_parts and not html_parts[-1].endswith(''):
# Only add break if not already after a closing tag
html_parts.append('
')
continue
# Check for markdown headers
if line.startswith('#'):
match = re.match(r'^(#+)\s*(.+)$', line)
if match:
level = min(len(match.group(1)), 6)
header_text = match.group(2).strip()
html_parts.append(f'{header_text}')
continue
# Check for blockquotes
if line.startswith('> '):
quote_text = line[2:].strip()
html_parts.append(f'{quote_text}
')
continue
# Check for lists
if re.match(r'^[*\-+]\s+', line):
list_text = re.sub(r'^[*\-+]\s+', '', line)
html_parts.append(f'{list_text}')
continue
if re.match(r'^\d+\.\s+', line):
list_text = re.sub(r'^\d+\.\s+', '', line)
html_parts.append(f'{list_text}')
continue
# Convert inline markdown
# Bold
line = re.sub(r'\*\*(.+?)\*\*', r'\1', line)
line = re.sub(r'__(.+?)__', r'\1', line)
# Italic
line = re.sub(r'\*(.+?)\*', r'\1', line)
line = re.sub(r'_(.+?)_', r'\1', line)
# Links
line = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', r'\1', line)
# Code inline
line = re.sub(r'`([^`]+)`', r'\1', line)
# Regular paragraph
html_parts.append(f'{line}
')
# Post-process lists to wrap in ul/ol tags
final_html = []
in_list = False
list_type = None
for part in html_parts:
if part.startswith(''):
if not in_list:
# Determine list type based on context (simplified)
list_type = 'ul' # Default to unordered
final_html.append(f'<{list_type}>')
in_list = True
final_html.append(part)
else:
if in_list:
final_html.append(f'')
in_list = False
final_html.append(part)
# Close any open list
if in_list:
final_html.append(f'')
return '\n'.join(final_html)
# =====================================================
# MAIN TRANSLATION FUNCTION
# =====================================================
def main(log_callback=None, stop_callback=None):
"""Main translation function with enhanced duplicate detection and progress tracking"""
config = TranslationConfig()
builtins._DISABLE_ZERO_DETECTION = config.DISABLE_ZERO_DETECTION
if config.DISABLE_ZERO_DETECTION:
print("=" * 60)
print("⚠️ 0-BASED DETECTION DISABLED BY USER")
print("⚠️ All chapter numbers will be used exactly as found")
print("=" * 60)
args = None
chapters_completed = 0
chunks_completed = 0
args = None
chapters_completed = 0
chunks_completed = 0
input_path = config.input_path
if not input_path and len(sys.argv) > 1:
input_path = sys.argv[1]
is_text_file = input_path.lower().endswith('.txt')
if is_text_file:
os.environ["IS_TEXT_FILE_TRANSLATION"] = "1"
import json as _json
_original_load = _json.load
def debug_json_load(fp, *args, **kwargs):
result = _original_load(fp, *args, **kwargs)
if isinstance(result, list) and len(result) > 0:
if isinstance(result[0], dict) and 'original_name' in result[0]:
print(f"[DEBUG] Loaded glossary list with {len(result)} items from {fp.name if hasattr(fp, 'name') else 'unknown'}")
return result
_json.load = debug_json_load
if log_callback:
set_output_redirect(log_callback)
def check_stop():
if stop_callback and stop_callback():
print("❌ Translation stopped by user request.")
return True
return is_stop_requested()
if config.EMERGENCY_RESTORE:
print("✅ Emergency paragraph restoration is ENABLED")
else:
print("⚠️ Emergency paragraph restoration is DISABLED")
print(f"[DEBUG] REMOVE_AI_ARTIFACTS environment variable: {os.getenv('REMOVE_AI_ARTIFACTS', 'NOT SET')}")
print(f"[DEBUG] REMOVE_AI_ARTIFACTS parsed value: {config.REMOVE_AI_ARTIFACTS}")
if config.REMOVE_AI_ARTIFACTS:
print("⚠️ AI artifact removal is ENABLED - will clean AI response artifacts")
else:
print("✅ AI artifact removal is DISABLED - preserving all content as-is")
if '--epub' in sys.argv or (len(sys.argv) > 1 and sys.argv[1].endswith(('.epub', '.txt'))):
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('epub', help='Input EPUB or text file')
args = parser.parse_args()
input_path = args.epub
is_text_file = input_path.lower().endswith('.txt')
if is_text_file:
file_base = os.path.splitext(os.path.basename(input_path))[0]
else:
epub_base = os.path.splitext(os.path.basename(input_path))[0]
file_base = epub_base
out = file_base
os.makedirs(out, exist_ok=True)
print(f"[DEBUG] Created output folder → {out}")
cleanup_previous_extraction(out)
os.environ["EPUB_OUTPUT_DIR"] = out
payloads_dir = out
# clear history if CONTEXTUAL is disabled
if not config.CONTEXTUAL:
history_file = os.path.join(payloads_dir, "translation_history.json")
if os.path.exists(history_file):
os.remove(history_file)
print("[DEBUG] CONTEXTUAL disabled - cleared translation history")
history_manager = HistoryManager(payloads_dir)
chapter_splitter = ChapterSplitter(model_name=config.MODEL)
chunk_context_manager = ChunkContextManager()
progress_manager = ProgressManager(payloads_dir)
# Create ChapterExtractor with progress callback if available
chapter_progress_callback = None
if log_callback:
# Create a wrapper that formats progress messages for the log
def chapter_progress_callback(msg):
log_callback(f"📊 {msg}")
chapter_extractor = ChapterExtractor(progress_callback=chapter_progress_callback)
glossary_manager = GlossaryManager()
history_file = os.path.join(payloads_dir, "translation_history.json")
if os.path.exists(history_file):
os.remove(history_file)
print(f"[DEBUG] Purged translation history → {history_file}")
print("🔍 Checking for deleted output files...")
progress_manager.cleanup_missing_files(out)
progress_manager.save()
if check_stop():
return
if not config.API_KEY:
print("❌ Error: Set API_KEY, OPENAI_API_KEY, or OPENAI_OR_Gemini_API_KEY in your environment.")
return
#print(f"[DEBUG] Found API key: {config.API_KEY[:10]}...")
print(f"[DEBUG] Using model = {config.MODEL}")
print(f"[DEBUG] Max output tokens = {config.MAX_OUTPUT_TOKENS}")
client = UnifiedClient(model=config.MODEL, api_key=config.API_KEY, output_dir=out)
if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
stats = client.get_stats()
print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
print(f" Active keys: {stats.get('active_keys', 0)}")
else:
print(f"🔑 Single-key mode: Using {config.MODEL}")
# Reset cleanup state when starting new translation
if hasattr(client, 'reset_cleanup_state'):
client.reset_cleanup_state()
if is_text_file:
print("📄 Processing text file...")
try:
txt_processor = TextFileProcessor(input_path, out)
chapters = txt_processor.extract_chapters()
txt_processor.save_original_structure()
metadata = {
"title": os.path.splitext(os.path.basename(input_path))[0],
"type": "text",
"chapter_count": len(chapters)
}
except ImportError as e:
print(f"❌ Error: Text file processor not available: {e}")
if log_callback:
log_callback(f"❌ Error: Text file processor not available: {e}")
return
except Exception as e:
print(f"❌ Error processing text file: {e}")
if log_callback:
log_callback(f"❌ Error processing text file: {e}")
return
else:
# Check if we should use async extraction (for GUI mode)
use_async_extraction = os.getenv("USE_ASYNC_CHAPTER_EXTRACTION", "0") == "1"
if use_async_extraction and log_callback:
print("🚀 Using async chapter extraction (subprocess mode)...")
from chapter_extraction_manager import ChapterExtractionManager
# Create manager with log callback
extraction_manager = ChapterExtractionManager(log_callback=log_callback)
# Get extraction mode
extraction_mode = os.getenv("EXTRACTION_MODE", "smart").lower()
# Define completion callback
extraction_result = {"completed": False, "result": None}
def on_extraction_complete(result):
extraction_result["completed"] = True
extraction_result["result"] = result
# Safety check for None result
if result is None:
log_callback("❌ Chapter extraction failed: No result returned")
return
if result.get("success"):
log_callback(f"✅ Chapter extraction completed: {result.get('chapters', 0)} chapters")
else:
log_callback(f"❌ Chapter extraction failed: {result.get('error', 'Unknown error')}")
# Start async extraction
extraction_manager.extract_chapters_async(
input_path,
out,
extraction_mode=extraction_mode,
progress_callback=lambda msg: log_callback(f"📊 {msg}"),
completion_callback=on_extraction_complete
)
# Wait for completion (with timeout)
timeout = 300 # 5 minutes timeout
start_time = time.time()
while not extraction_result["completed"]:
if check_stop():
extraction_manager.stop_extraction()
return
if time.time() - start_time > timeout:
log_callback("⚠️ Chapter extraction timeout")
extraction_manager.stop_extraction()
return
time.sleep(0.1) # Check every 100ms
# Check if extraction was successful
if not extraction_result["result"] or not extraction_result["result"].get("success"):
log_callback("❌ Chapter extraction failed")
return
# Load the extracted data
metadata_path = os.path.join(out, "metadata.json")
if os.path.exists(metadata_path):
with open(metadata_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
else:
metadata = extraction_result["result"].get("metadata", {})
# The async extraction should have saved chapters directly, similar to the sync version
# We need to reconstruct the chapters list with body content
# Check if the extraction actually created a chapters.json file with full content
chapters_full_path = os.path.join(out, "chapters_full.json")
chapters_info_path = os.path.join(out, "chapters_info.json")
chapters = []
# First try to load full chapters if saved
if os.path.exists(chapters_full_path):
log_callback("Loading full chapters data...")
with open(chapters_full_path, 'r', encoding='utf-8') as f:
chapters = json.load(f)
log_callback(f"✅ Loaded {len(chapters)} chapters with content")
elif os.path.exists(chapters_info_path):
# Fall back to loading from individual files
log_callback("Loading chapter info and searching for content files...")
with open(chapters_info_path, 'r', encoding='utf-8') as f:
chapters_info = json.load(f)
# List all files in the output directory
all_files = os.listdir(out)
log_callback(f"Found {len(all_files)} files in output directory")
# Try to match chapter files
for info in chapters_info:
chapter_num = info['num']
found = False
# Try different naming patterns
patterns = [
f"chapter_{chapter_num:04d}_", # With leading zeros
f"chapter_{chapter_num}_", # Without leading zeros
f"ch{chapter_num:04d}_", # Shortened with zeros
f"ch{chapter_num}_", # Shortened without zeros
f"{chapter_num:04d}_", # Just number with zeros
f"{chapter_num}_" # Just number
]
for pattern in patterns:
# Find files matching this pattern (any extension)
matching_files = [f for f in all_files if f.startswith(pattern)]
if matching_files:
# Prefer HTML/XHTML files
html_files = [f for f in matching_files if f.endswith(('.html', '.xhtml', '.htm'))]
if html_files:
chapter_file = html_files[0]
else:
chapter_file = matching_files[0]
chapter_path = os.path.join(out, chapter_file)
try:
with open(chapter_path, 'r', encoding='utf-8') as f:
content = f.read()
chapters.append({
"num": chapter_num,
"title": info.get("title", f"Chapter {chapter_num}"),
"body": content,
"filename": info.get("original_filename", ""),
"has_images": info.get("has_images", False),
"file_size": len(content),
"content_hash": info.get("content_hash", "")
})
found = True
break
except Exception as e:
log_callback(f"⚠️ Error reading {chapter_file}: {e}")
if not found:
log_callback(f"⚠️ No file found for Chapter {chapter_num}")
# Log available files for debugging
if len(all_files) < 50:
similar_files = [f for f in all_files if str(chapter_num) in f]
if similar_files:
log_callback(f" Similar files: {similar_files[:3]}")
if not chapters:
log_callback("❌ No chapters could be loaded!")
log_callback(f"❌ Output directory: {out}")
log_callback(f"❌ Files in directory: {len(os.listdir(out))} files")
# Show first few files for debugging
sample_files = os.listdir(out)[:10]
log_callback(f"❌ Sample files: {sample_files}")
return
# Sort chapters by OPF spine order if available
opf_path = os.path.join(out, 'content.opf')
if os.path.exists(opf_path) and chapters:
log_callback("📋 Sorting chapters according to OPF spine order...")
# Use the existing chapter_extractor instance to sort
chapters = chapter_extractor._sort_by_opf_spine(chapters, opf_path)
log_callback("✅ Chapters sorted according to OPF reading order")
else:
print("🚀 Using comprehensive chapter extraction with resource handling...")
with zipfile.ZipFile(input_path, 'r') as zf:
metadata = chapter_extractor._extract_epub_metadata(zf)
chapters = chapter_extractor.extract_chapters(zf, out)
print(f"\n📚 Extraction Summary:")
print(f" Total chapters extracted: {len(chapters)}")
if chapters:
nums = [c.get('num', 0) for c in chapters]
print(f" Chapter range: {min(nums)} to {max(nums)}")
# Check for gaps in the sequence
expected_count = max(nums) - min(nums) + 1
if len(chapters) < expected_count:
print(f"\n⚠️ Potential missing chapters detected:")
print(f" Expected {expected_count} chapters (from {min(nums)} to {max(nums)})")
print(f" Actually found: {len(chapters)} chapters")
print(f" Potentially missing: {expected_count - len(chapters)} chapters")
validate_chapter_continuity(chapters)
print("\n" + "="*50)
validate_epub_structure(out)
print("="*50 + "\n")
progress_manager.migrate_to_content_hash(chapters)
progress_manager.save()
if check_stop():
return
metadata_path = os.path.join(out, "metadata.json")
if os.path.exists(metadata_path):
with open(metadata_path, 'r', encoding='utf-8') as mf:
metadata = json.load(mf)
metadata["chapter_count"] = len(chapters)
metadata["chapter_titles"] = {str(c["num"]): c["title"] for c in chapters}
print(f"[DEBUG] Initializing client with model = {config.MODEL}")
client = UnifiedClient(api_key=config.API_KEY, model=config.MODEL, output_dir=out)
if hasattr(client, 'use_multi_keys') and client.use_multi_keys:
stats = client.get_stats()
print(f"🔑 Multi-key mode active: {stats.get('total_keys', 0)} keys loaded")
print(f" Active keys: {stats.get('active_keys', 0)}")
else:
print(f"🔑 Single-key mode: Using {config.MODEL}")
# Reset cleanup state when starting new translation
if hasattr(client, 'reset_cleanup_state'):
client.reset_cleanup_state()
if "title" in metadata and config.TRANSLATE_BOOK_TITLE and not metadata.get("title_translated", False):
original_title = metadata["title"]
print(f"📚 Original title: {original_title}")
if not check_stop():
translated_title = translate_title(
original_title,
client,
None,
None,
config.TEMP
)
metadata["original_title"] = original_title
metadata["title"] = translated_title
metadata["title_translated"] = True
print(f"📚 Translated title: {translated_title}")
else:
print("❌ Title translation skipped due to stop request")
# Translate other metadata fields if configured
translate_metadata_fields_str = os.getenv('TRANSLATE_METADATA_FIELDS', '{}')
metadata_translation_mode = os.getenv('METADATA_TRANSLATION_MODE', 'together')
try:
translate_metadata_fields = json.loads(translate_metadata_fields_str)
if translate_metadata_fields and any(translate_metadata_fields.values()):
# Filter out fields that should be translated (excluding already translated fields)
fields_to_translate = {}
skipped_fields = []
for field_name, should_translate in translate_metadata_fields.items():
if should_translate and field_name != 'title' and field_name in metadata:
# Check if already translated
if metadata.get(f"{field_name}_translated", False):
skipped_fields.append(field_name)
print(f"✓ Skipping {field_name} - already translated")
else:
fields_to_translate[field_name] = should_translate
if fields_to_translate:
print("\n" + "="*50)
print("📋 METADATA TRANSLATION PHASE")
print("="*50)
print(f"🌐 Translating {len(fields_to_translate)} metadata fields...")
# Get ALL configuration from environment - NO DEFAULTS
system_prompt = os.getenv('BOOK_TITLE_SYSTEM_PROMPT', '')
if not system_prompt:
print("❌ No system prompt configured, skipping metadata translation")
else:
# Get field-specific prompts
field_prompts_str = os.getenv('METADATA_FIELD_PROMPTS', '{}')
try:
field_prompts = json.loads(field_prompts_str)
except:
field_prompts = {}
if not field_prompts and not field_prompts.get('_default'):
print("❌ No field prompts configured, skipping metadata translation")
else:
# Get language configuration
lang_behavior = os.getenv('LANG_PROMPT_BEHAVIOR', 'auto')
forced_source_lang = os.getenv('FORCED_SOURCE_LANG', 'Korean')
output_language = os.getenv('OUTPUT_LANGUAGE', 'English')
# Determine source language
source_lang = metadata.get('language', '').lower()
if lang_behavior == 'never':
lang_str = ""
elif lang_behavior == 'always':
lang_str = forced_source_lang
else: # auto
if 'zh' in source_lang or 'chinese' in source_lang:
lang_str = 'Chinese'
elif 'ja' in source_lang or 'japanese' in source_lang:
lang_str = 'Japanese'
elif 'ko' in source_lang or 'korean' in source_lang:
lang_str = 'Korean'
else:
lang_str = ''
# Check if batch translation is enabled for parallel processing
batch_translate_enabled = os.getenv('BATCH_TRANSLATION', '0') == '1'
batch_size = int(os.getenv('BATCH_SIZE', '50')) # Default batch size
if batch_translate_enabled and len(fields_to_translate) > 1:
print(f"⚡ Using parallel metadata translation mode ({len(fields_to_translate)} fields, batch size: {batch_size})...")
# Import ThreadPoolExecutor for parallel processing
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading
# Thread-safe results storage
translation_results = {}
results_lock = threading.Lock()
def translate_metadata_field(field_name, original_value):
"""Translate a single metadata field"""
try:
print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
# Get field-specific prompt
prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
if not prompt_template:
print(f"⚠️ No prompt configured for field '{field_name}', skipping")
return None
# Replace variables in prompt
field_prompt = prompt_template.replace('{source_lang}', lang_str)
field_prompt = field_prompt.replace('{output_lang}', output_language)
field_prompt = field_prompt.replace('English', output_language)
field_prompt = field_prompt.replace('{field_value}', str(original_value))
# Check if we're using a translation service (not AI)
client_type = getattr(client, 'client_type', '')
is_translation_service = client_type in ['deepl', 'google_translate']
if is_translation_service:
# For translation services, send only the field value without AI prompts
print(f"🌐 Using translation service ({client_type}) - sending field directly")
messages = [
{"role": "user", "content": str(original_value)}
]
else:
# For AI services, use prompts as before
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
]
# Add delay for rate limiting
if config.DELAY > 0:
time.sleep(config.DELAY)
# Make API call
content, finish_reason = client.send(
messages,
temperature=config.TEMP,
max_tokens=config.MAX_OUTPUT_TOKENS
)
translated_value = content.strip()
# Store result thread-safely
with results_lock:
translation_results[field_name] = {
'original': original_value,
'translated': translated_value,
'success': True
}
print(f"✅ Translated {field_name}: {translated_value}")
return translated_value
except Exception as e:
print(f"❌ Failed to translate {field_name}: {e}")
with results_lock:
translation_results[field_name] = {
'original': original_value,
'translated': None,
'success': False,
'error': str(e)
}
return None
# Execute parallel translations with limited workers
max_workers = min(len(fields_to_translate), batch_size)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all translation tasks
futures = {}
for field_name in fields_to_translate:
if field_name in metadata and not check_stop():
original_value = metadata[field_name]
future = executor.submit(translate_metadata_field, field_name, original_value)
futures[future] = field_name
# Wait for completion
for future in as_completed(futures):
if check_stop():
print("❌ Metadata translation stopped by user")
break
# Apply results to metadata
for field_name, result in translation_results.items():
if result['success'] and result['translated']:
metadata[f"original_{field_name}"] = result['original']
metadata[field_name] = result['translated']
metadata[f"{field_name}_translated"] = True
else:
# Sequential translation mode (individual translation)
mode_desc = "sequential" if not batch_translate_enabled else "sequential (single field)"
print(f"📝 Using {mode_desc} translation mode...")
for field_name in fields_to_translate:
if not check_stop() and field_name in metadata:
original_value = metadata[field_name]
print(f"\n📋 Translating {field_name}: {original_value[:100]}..."
if len(str(original_value)) > 100 else f"\n📋 Translating {field_name}: {original_value}")
# Get field-specific prompt
prompt_template = field_prompts.get(field_name, field_prompts.get('_default', ''))
if not prompt_template:
print(f"⚠️ No prompt configured for field '{field_name}', skipping")
continue
# Replace variables in prompt
field_prompt = prompt_template.replace('{source_lang}', lang_str)
field_prompt = field_prompt.replace('{output_lang}', output_language)
field_prompt = field_prompt.replace('English', output_language)
field_prompt = field_prompt.replace('{field_value}', str(original_value))
# Check if we're using a translation service (not AI)
client_type = getattr(client, 'client_type', '')
is_translation_service = client_type in ['deepl', 'google_translate']
if is_translation_service:
# For translation services, send only the field value without AI prompts
print(f"🌐 Using translation service ({client_type}) - sending field directly")
messages = [
{"role": "user", "content": str(original_value)}
]
else:
# For AI services, use prompts as before
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"{field_prompt}\n\n{original_value}"}
]
try:
# Add delay using the config instance from main()
if config.DELAY > 0: # ✅ FIXED - use config.DELAY instead of config.SEND_INTERVAL
time.sleep(config.DELAY)
# Use the same client instance from main()
# ✅ FIXED - Properly unpack tuple response and provide max_tokens
content, finish_reason = client.send(
messages,
temperature=config.TEMP,
max_tokens=config.MAX_OUTPUT_TOKENS # ✅ FIXED - provide max_tokens to avoid NoneType error
)
translated_value = content.strip() # ✅ FIXED - use content from unpacked tuple
metadata[f"original_{field_name}"] = original_value
metadata[field_name] = translated_value
metadata[f"{field_name}_translated"] = True
print(f"✅ Translated {field_name}: {translated_value}")
except Exception as e:
print(f"❌ Failed to translate {field_name}: {e}")
else:
if check_stop():
print("❌ Metadata translation stopped by user")
break
else:
print("📋 No additional metadata fields to translate")
except Exception as e:
print(f"⚠️ Error processing metadata translation settings: {e}")
import traceback
traceback.print_exc()
with open(metadata_path, 'w', encoding='utf-8') as mf:
json.dump(metadata, mf, ensure_ascii=False, indent=2)
print(f"💾 Saved metadata with {'translated' if metadata.get('title_translated', False) else 'original'} title")
print("\n" + "="*50)
print("📑 GLOSSARY GENERATION PHASE")
print("="*50)
print(f"📑 DEBUG: ENABLE_AUTO_GLOSSARY = '{os.getenv('ENABLE_AUTO_GLOSSARY', 'NOT SET')}'")
print(f"📑 DEBUG: MANUAL_GLOSSARY = '{config.MANUAL_GLOSSARY}'")
print(f"📑 DEBUG: Manual glossary exists? {os.path.isfile(config.MANUAL_GLOSSARY) if config.MANUAL_GLOSSARY else False}")
# Check if glossary.csv already exists in the source folder
existing_glossary_csv = os.path.join(out, "glossary.csv")
existing_glossary_json = os.path.join(out, "glossary.json")
print(f"📑 DEBUG: Existing glossary.csv? {os.path.exists(existing_glossary_csv)}")
print(f"📑 DEBUG: Existing glossary.json? {os.path.exists(existing_glossary_json)}")
if config.MANUAL_GLOSSARY and os.path.isfile(config.MANUAL_GLOSSARY):
ext = os.path.splitext(config.MANUAL_GLOSSARY)[1].lower()
target_name = "glossary.csv" if ext == ".csv" else "glossary.json"
target_path = os.path.join(out, target_name)
if os.path.abspath(config.MANUAL_GLOSSARY) != os.path.abspath(target_path):
shutil.copy(config.MANUAL_GLOSSARY, target_path)
print("📑 Using manual glossary from:", config.MANUAL_GLOSSARY)
else:
print("📑 Using existing glossary:", config.MANUAL_GLOSSARY)
elif os.path.exists(existing_glossary_csv) or os.path.exists(existing_glossary_json):
print("📑 Existing glossary file detected in source folder - skipping automatic generation")
if os.path.exists(existing_glossary_csv):
print(f"📑 Using existing glossary.csv: {existing_glossary_csv}")
elif os.path.exists(existing_glossary_json):
print(f"📑 Using existing glossary.json: {existing_glossary_json}")
elif os.getenv("ENABLE_AUTO_GLOSSARY", "0") == "1":
model = os.getenv("MODEL", "gpt-4")
if is_traditional_translation_api(model):
print("📑 Automatic glossary generation disabled")
print(f" {model} does not support glossary extraction")
print(" Traditional translation APIs cannot identify character names/terms")
else:
print("📑 Starting automatic glossary generation...")
try:
# Use the new process-safe glossary worker
from glossary_process_worker import generate_glossary_in_process
import concurrent.futures
import multiprocessing
instructions = ""
# Get extraction workers setting
extraction_workers = int(os.getenv("EXTRACTION_WORKERS", "1"))
if extraction_workers == 1:
# Auto-detect for better performance
extraction_workers = min(os.cpu_count() or 4, 4)
print(f"📑 Using {extraction_workers} CPU cores for glossary generation")
# Collect environment variables to pass to subprocess
env_vars = {}
important_vars = [
'EXTRACTION_WORKERS', 'GLOSSARY_MIN_FREQUENCY', 'GLOSSARY_MAX_NAMES',
'GLOSSARY_MAX_TITLES', 'GLOSSARY_BATCH_SIZE', 'GLOSSARY_STRIP_HONORIFICS',
'GLOSSARY_FUZZY_THRESHOLD', 'GLOSSARY_MAX_TEXT_SIZE', 'AUTO_GLOSSARY_PROMPT',
'GLOSSARY_USE_SMART_FILTER', 'GLOSSARY_USE_LEGACY_CSV', 'GLOSSARY_PARALLEL_ENABLED',
'GLOSSARY_FILTER_MODE', 'GLOSSARY_SKIP_FREQUENCY_CHECK', 'GLOSSARY_SKIP_ALL_VALIDATION',
'MODEL', 'API_KEY', 'OPENAI_API_KEY', 'GEMINI_API_KEY', 'MAX_OUTPUT_TOKENS',
'GLOSSARY_TEMPERATURE', 'MANUAL_GLOSSARY', 'ENABLE_AUTO_GLOSSARY'
]
for var in important_vars:
if var in os.environ:
env_vars[var] = os.environ[var]
# Create a Queue for real-time log streaming
manager = multiprocessing.Manager()
log_queue = manager.Queue()
# Use ProcessPoolExecutor for true parallelism (completely bypasses GIL)
print("📑 Starting glossary generation in separate process...")
with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
# Submit to separate process WITH log queue
future = executor.submit(
generate_glossary_in_process,
out,
chapters,
instructions,
env_vars,
log_queue # Pass the queue for real-time logs
)
# Poll for completion and stream logs in real-time
poll_count = 0
while not future.done():
poll_count += 1
# Check for logs from subprocess and print them immediately
try:
while not log_queue.empty():
log_line = log_queue.get_nowait()
print(log_line) # Print to GUI
except:
pass
# Super short sleep to yield to GUI
time.sleep(0.001)
# Check for stop every 100 polls
if poll_count % 100 == 0:
if check_stop():
print("📑 ❌ Glossary generation cancelled")
executor.shutdown(wait=False, cancel_futures=True)
return
# Get any remaining logs from queue
try:
while not log_queue.empty():
log_line = log_queue.get_nowait()
print(log_line)
except:
pass
# Get result
if future.done():
try:
result = future.result(timeout=0.1)
if isinstance(result, dict):
if result.get('success'):
print(f"📑 ✅ Glossary generation completed successfully")
else:
print(f"📑 ❌ Glossary generation failed: {result.get('error')}")
if result.get('traceback'):
print(f"📑 Error details:\n{result.get('traceback')}")
except Exception as e:
print(f"📑 ❌ Error retrieving glossary result: {e}")
print("✅ Automatic glossary generation COMPLETED")
# Handle deferred glossary appending
if os.getenv('DEFER_GLOSSARY_APPEND') == '1':
print("📑 Processing deferred glossary append to system prompt...")
glossary_path = find_glossary_file(out)
if glossary_path and os.path.exists(glossary_path):
try:
glossary_block = None
if glossary_path.lower().endswith('.csv'):
with open(glossary_path, 'r', encoding='utf-8') as f:
glossary_block = f.read()
else:
with open(glossary_path, 'r', encoding='utf-8') as f:
glossary_data = json.load(f)
formatted_entries = {}
if isinstance(glossary_data, dict) and 'entries' in glossary_data:
formatted_entries = glossary_data['entries']
elif isinstance(glossary_data, dict):
formatted_entries = {k: v for k, v in glossary_data.items() if k != "metadata"}
if formatted_entries:
glossary_block = json.dumps(formatted_entries, ensure_ascii=False, indent=2)
else:
glossary_block = None
if glossary_block:
glossary_prompt = os.getenv('GLOSSARY_APPEND_PROMPT',
"Character/Term Glossary (use these translations consistently):")
current_prompt = config.PROMPT
if current_prompt:
current_prompt += "\n\n"
current_prompt += f"{glossary_prompt}\n{glossary_block}"
config.PROMPT = current_prompt
print(f"✅ Added auto-generated glossary to system prompt ({os.path.basename(glossary_path)})")
if 'DEFER_GLOSSARY_APPEND' in os.environ:
del os.environ['DEFER_GLOSSARY_APPEND']
if 'GLOSSARY_APPEND_PROMPT' in os.environ:
del os.environ['GLOSSARY_APPEND_PROMPT']
else:
print("⚠️ Auto-generated glossary has no entries - skipping append")
if 'DEFER_GLOSSARY_APPEND' in os.environ:
del os.environ['DEFER_GLOSSARY_APPEND']
if 'GLOSSARY_APPEND_PROMPT' in os.environ:
del os.environ['GLOSSARY_APPEND_PROMPT']
except Exception as e:
print(f"⚠️ Failed to append auto-generated glossary: {e}")
else:
print("⚠️ No glossary file found after automatic generation")
except Exception as e:
print(f"❌ Glossary generation failed: {e}")
else:
print("📑 Automatic glossary generation disabled")
# Don't create an empty glossary - let any existing manual glossary remain
glossary_file = find_glossary_file(out)
if glossary_file and os.path.exists(glossary_file):
try:
if glossary_file.lower().endswith('.csv'):
# Quick CSV stats
with open(glossary_file, 'r', encoding='utf-8') as f:
lines = [ln.strip() for ln in f.readlines() if ln.strip()]
entry_count = max(0, len(lines) - 1) if lines and ',' in lines[0] else len(lines)
print(f"📑 Glossary ready (CSV) with {entry_count} entries")
print("📑 Sample glossary lines:")
for ln in lines[1:4]:
print(f" • {ln}")
else:
with open(glossary_file, 'r', encoding='utf-8') as f:
glossary_data = json.load(f)
if isinstance(glossary_data, dict):
if 'entries' in glossary_data and isinstance(glossary_data['entries'], dict):
entry_count = len(glossary_data['entries'])
sample_items = list(glossary_data['entries'].items())[:3]
else:
entry_count = len(glossary_data)
sample_items = list(glossary_data.items())[:3]
print(f"📑 Glossary ready with {entry_count} entries")
print("📑 Sample glossary entries:")
for key, value in sample_items:
print(f" • {key} → {value}")
elif isinstance(glossary_data, list):
print(f"📑 Glossary ready with {len(glossary_data)} entries")
print("📑 Sample glossary entries:")
for i, entry in enumerate(glossary_data[:3]):
if isinstance(entry, dict):
original = entry.get('original_name', '?')
translated = entry.get('name', original)
print(f" • {original} → {translated}")
else:
print(f"⚠️ Unexpected glossary format: {type(glossary_data)}")
except Exception as e:
print(f"⚠️ Failed to inspect glossary file: {e}")
else:
print("📑 No glossary file found")
print("="*50)
print("🚀 STARTING MAIN TRANSLATION PHASE")
print("="*50 + "\n")
glossary_path = find_glossary_file(out)
if glossary_path and os.path.exists(glossary_path) and glossary_path.lower().endswith('.json'):
try:
with open(glossary_path, 'r', encoding='utf-8') as f:
g_data = json.load(f)
print(f"[DEBUG] Glossary type before translation: {type(g_data)}")
if isinstance(g_data, list):
print(f"[DEBUG] Glossary is a list")
except Exception as e:
print(f"[DEBUG] Error checking glossary: {e}")
glossary_path = find_glossary_file(out)
system = build_system_prompt(config.SYSTEM_PROMPT, glossary_path)
base_msg = [{"role": "system", "content": system}]
# Preserve the original system prompt to avoid in-place mutations
original_system_prompt = system
last_summary_block_text = None # Will hold the last rolling summary text for the NEXT chapter only
image_translator = None
if config.ENABLE_IMAGE_TRANSLATION:
print(f"🖼️ Image translation enabled for model: {config.MODEL}")
print("🖼️ Image translation will use your custom system prompt and glossary")
image_translator = ImageTranslator(
client,
out,
config.PROFILE_NAME,
system,
config.TEMP,
log_callback ,
progress_manager,
history_manager,
chunk_context_manager
)
known_vision_models = [
'gemini-1.5-pro', 'gemini-1.5-flash', 'gemini-2.0-flash', 'gemini-2.5-flash', 'gemini-2.5-pro',
'gpt-4-turbo', 'gpt-4o', 'gpt-4.1-mini', 'gpt-4.1-nano', 'o4-mini', 'gpt-4.1-mini'
]
if config.MODEL.lower() not in known_vision_models:
print(f"⚠️ Note: {config.MODEL} may not have vision capabilities. Image translation will be attempted anyway.")
else:
print("ℹ️ Image translation disabled by user")
total_chapters = len(chapters)
# Only detect numbering if the toggle is not disabled
if config.DISABLE_ZERO_DETECTION:
print(f"📊 0-based detection disabled by user setting")
uses_zero_based = False
# Important: Set a flag that can be checked throughout the codebase
config._force_disable_zero_detection = True
else:
if chapters:
uses_zero_based = detect_novel_numbering(chapters)
print(f"📊 Novel numbering detected: {'0-based' if uses_zero_based else '1-based'}")
else:
uses_zero_based = False
config._force_disable_zero_detection = False
# Store this for later use
config._uses_zero_based = uses_zero_based
rng = os.getenv("CHAPTER_RANGE", "")
start = None
end = None
if rng and re.match(r"^\d+\s*-\s*\d+$", rng):
start, end = map(int, rng.split("-", 1))
if config.DISABLE_ZERO_DETECTION:
print(f"📊 0-based detection disabled - using range as specified: {start}-{end}")
elif uses_zero_based:
print(f"📊 0-based novel detected")
print(f"📊 User range {start}-{end} will be used as-is (chapters are already adjusted)")
else:
print(f"📊 1-based novel detected")
print(f"📊 Using range as specified: {start}-{end}")
print("📊 Calculating total chunks needed...")
total_chunks_needed = 0
chunks_per_chapter = {}
chapters_to_process = 0
# When setting actual chapter numbers (in the main function)
for idx, c in enumerate(chapters):
chap_num = c["num"]
content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
# Extract the raw chapter number from the file
raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
#print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
# Apply the offset
offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
raw_num += offset
# When toggle is disabled, use raw numbers without any 0-based adjustment
if config.DISABLE_ZERO_DETECTION:
c['actual_chapter_num'] = raw_num
# Store raw number for consistency
c['raw_chapter_num'] = raw_num
c['zero_adjusted'] = False
else:
# Store raw number
c['raw_chapter_num'] = raw_num
# Apply adjustment only if this is a 0-based novel
if uses_zero_based:
c['actual_chapter_num'] = raw_num + 1
c['zero_adjusted'] = True
else:
c['actual_chapter_num'] = raw_num
c['zero_adjusted'] = False
# Now we can safely use actual_num
actual_num = c['actual_chapter_num']
if start is not None:
if not (start <= c['actual_chapter_num'] <= end):
#print(f"[SKIP] Chapter {c['actual_chapter_num']} outside range {start}-{end}")
continue
needs_translation, skip_reason, _ = progress_manager.check_chapter_status(
idx, actual_num, content_hash, out
)
if not needs_translation:
chunks_per_chapter[idx] = 0
continue
chapters_to_process += 1
chapter_key = str(actual_num)
if chapter_key in progress_manager.prog["chapters"] and progress_manager.prog["chapters"][chapter_key].get("status") == "in_progress":
pass
# Calculate based on OUTPUT limit only
max_output_tokens = config.MAX_OUTPUT_TOKENS
safety_margin_output = 500
# Korean to English typically compresses to 0.7-0.9x
compression_factor = config.COMPRESSION_FACTOR
available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
# Ensure minimum
available_tokens = max(available_tokens, 1000)
#print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
# For mixed content chapters, calculate on clean text
# For mixed content chapters, calculate on clean text
if c.get('has_images', False) and ContentProcessor.is_meaningful_text_content(c["body"]):
# Don't modify c["body"] at all during chunk calculation
# Just pass the body as-is, the chunking will be slightly off but that's OK
chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
else:
chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
chapter_key_str = content_hash
old_key_str = str(idx)
if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
del progress_manager.prog["chapter_chunks"][old_key_str]
#print(f"[PROGRESS] Migrated chunks for chapter {actual_num} to new tracking system")
# Always count actual chunks - ignore "completed" tracking
chunks_per_chapter[idx] = len(chunks)
total_chunks_needed += chunks_per_chapter[idx]
terminology = "Sections" if is_text_file else "Chapters"
print(f"📊 Total chunks to translate: {total_chunks_needed}")
print(f"📚 {terminology} to process: {chapters_to_process}")
multi_chunk_chapters = [(idx, count) for idx, count in chunks_per_chapter.items() if count > 1]
if multi_chunk_chapters:
# Determine terminology based on file type
terminology = "Sections" if is_text_file else "Chapters"
print(f"📄 {terminology} requiring multiple chunks:")
for idx, chunk_count in multi_chunk_chapters:
chap = chapters[idx]
section_term = "Section" if is_text_file else "Chapter"
print(f" • {section_term} {idx+1} ({chap['title'][:30]}...): {chunk_count} chunks")
translation_start_time = time.time()
chunks_completed = 0
chapters_completed = 0
current_chunk_number = 0
if config.BATCH_TRANSLATION:
print(f"\n📦 PARALLEL TRANSLATION MODE ENABLED")
print(f"📦 Processing chapters with up to {config.BATCH_SIZE} concurrent API calls")
import concurrent.futures
from threading import Lock
progress_lock = Lock()
chapters_to_translate = []
# FIX: First pass to set actual chapter numbers for ALL chapters
# This ensures batch mode has the same chapter numbering as non-batch mode
print("📊 Setting chapter numbers...")
for idx, c in enumerate(chapters):
raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
# Apply offset if configured
offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
raw_num += offset
if config.DISABLE_ZERO_DETECTION:
# Use raw numbers without adjustment
c['actual_chapter_num'] = raw_num
c['raw_chapter_num'] = raw_num
c['zero_adjusted'] = False
else:
# Store raw number
c['raw_chapter_num'] = raw_num
# Apply 0-based adjustment if detected
if uses_zero_based:
c['actual_chapter_num'] = raw_num + 1
c['zero_adjusted'] = True
else:
c['actual_chapter_num'] = raw_num
c['zero_adjusted'] = False
for idx, c in enumerate(chapters):
chap_num = c["num"]
content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
# Check if this is a pre-split text chunk with decimal number
if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
actual_num = c['num'] # Preserve the decimal for text files only
else:
actual_num = c.get('actual_chapter_num', c['num']) # Now this will exist!
# Skip chapters outside the range
if start is not None and not (start <= actual_num <= end):
continue
# Check if chapter needs translation
needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
idx, actual_num, content_hash, out, c # Pass the chapter object
)
# Add explicit file check for supposedly completed chapters
if not needs_translation and existing_file:
file_path = os.path.join(out, existing_file)
if not os.path.exists(file_path):
print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
needs_translation = True
skip_reason = None
# Update status to file_missing
progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
progress_manager.save()
if not needs_translation:
# Modify skip_reason to use appropriate terminology
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
# Replace "Chapter" with appropriate terminology in skip_reason
skip_reason_modified = skip_reason.replace("Chapter", terminology)
print(f"[SKIP] {skip_reason_modified}")
chapters_completed += 1
continue
# Check for empty or image-only chapters
has_images = c.get('has_images', False)
has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
text_size = c.get('file_size', 0)
is_empty_chapter = (not has_images and text_size < 10)
is_image_only_chapter = (has_images and not has_meaningful_text)
# Handle empty chapters
if is_empty_chapter:
print(f"📄 Empty chapter {chap_num} - will process individually")
safe_title = make_safe_filename(c['title'], c['num'])
if isinstance(c['num'], float):
fname = FileUtilities.create_chapter_filename(c, c['num'])
else:
fname = FileUtilities.create_chapter_filename(c, c['num'])
with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
f.write(c["body"])
progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
progress_manager.save()
chapters_completed += 1
continue
# Add to chapters to translate
chapters_to_translate.append((idx, c))
print(f"📊 Found {len(chapters_to_translate)} chapters to translate in parallel")
# Continue with the rest of the existing batch processing code...
batch_processor = BatchTranslationProcessor(
config, client, base_msg, out, progress_lock,
progress_manager.save,
lambda idx, actual_num, content_hash, output_file=None, status="completed", **kwargs: progress_manager.update(idx, actual_num, content_hash, output_file, status, **kwargs),
check_stop,
image_translator,
is_text_file=is_text_file
)
total_to_process = len(chapters_to_translate)
processed = 0
# Apply conservative batching setting
batch_multiplier = 3 if os.getenv('CONSERVATIVE_BATCHING', '0') == '1' else 1
batch_group_size = config.BATCH_SIZE * batch_multiplier
if batch_multiplier > 1:
print(f"📦 Using conservative batching: {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
else:
print(f"📦 Using direct batching (default): {batch_group_size} chapters per group, {config.BATCH_SIZE} parallel")
with concurrent.futures.ThreadPoolExecutor(max_workers=config.BATCH_SIZE) as executor:
for batch_start in range(0, total_to_process, batch_group_size):
if check_stop():
print("❌ Translation stopped during parallel processing")
executor.shutdown(wait=False)
return
batch_end = min(batch_start + batch_group_size, total_to_process)
current_batch = chapters_to_translate[batch_start:batch_end]
batch_number = (batch_start // batch_group_size) + 1
print(f"\n📦 Submitting batch {batch_number}: {len(current_batch)} chapters")
future_to_chapter = {
executor.submit(batch_processor.process_single_chapter, chapter_data): chapter_data
for chapter_data in current_batch
}
active_count = 0
completed_in_batch = 0
failed_in_batch = 0
for future in concurrent.futures.as_completed(future_to_chapter):
if check_stop():
print("❌ Translation stopped")
executor.shutdown(wait=False)
return
chapter_data = future_to_chapter[future]
idx, chapter = chapter_data
try:
success, chap_num = future.result()
if success:
completed_in_batch += 1
print(f"✅ Chapter {chap_num} done ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
else:
failed_in_batch += 1
print(f"❌ Chapter {chap_num} failed ({completed_in_batch + failed_in_batch}/{len(current_batch)} in batch)")
except Exception as e:
failed_in_batch += 1
print(f"❌ Chapter thread error: {e}")
processed += 1
progress_percent = (processed / total_to_process) * 100
print(f"📊 Overall Progress: {processed}/{total_to_process} ({progress_percent:.1f}%)")
print(f"\n📦 Batch Summary:")
print(f" ✅ Successful: {completed_in_batch}")
print(f" ❌ Failed: {failed_in_batch}")
if batch_end < total_to_process:
print(f"⏳ Waiting {config.DELAY}s before next batch...")
time.sleep(config.DELAY)
chapters_completed = batch_processor.chapters_completed
chunks_completed = batch_processor.chunks_completed
print(f"\n🎉 Parallel translation complete!")
print(f" Total chapters processed: {processed}")
# Count qa_failed chapters correctly
qa_failed_count = 0
actual_successful = 0
for idx, c in enumerate(chapters):
# Get the chapter's actual number
if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
actual_num = c['num']
else:
actual_num = c.get('actual_chapter_num', c['num'])
# Check if this chapter was processed and has qa_failed status
content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
# Check if this chapter exists in progress
chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
status = chapter_info.get("status")
if status == "qa_failed":
qa_failed_count += 1
elif status == "completed":
actual_successful += 1
# Correct the displayed counts
print(f" Successful: {actual_successful}")
if qa_failed_count > 0:
print(f"\n⚠️ {qa_failed_count} chapters failed due to content policy violations:")
qa_failed_chapters = []
for idx, c in enumerate(chapters):
if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
actual_num = c['num']
else:
actual_num = c.get('actual_chapter_num', c['num'])
content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
chapter_info = progress_manager.prog["chapters"].get(content_hash, {})
if chapter_info.get("status") == "qa_failed":
qa_failed_chapters.append(actual_num)
print(f" Failed chapters: {', '.join(map(str, sorted(qa_failed_chapters)))}")
# Stop translation completely after batch mode
print("\n📌 Batch translation completed.")
elif not config.BATCH_TRANSLATION:
translation_processor = TranslationProcessor(config, client, out, log_callback, check_stop, uses_zero_based, is_text_file)
if config.DUPLICATE_DETECTION_MODE == 'ai-hunter':
# Build the main config from environment variables and config object
main_config = {
'duplicate_lookback_chapters': config.DUPLICATE_LOOKBACK_CHAPTERS,
'duplicate_detection_mode': config.DUPLICATE_DETECTION_MODE,
}
# Check if AI Hunter config was passed via environment variable
ai_hunter_config_str = os.getenv('AI_HUNTER_CONFIG')
if ai_hunter_config_str:
try:
ai_hunter_config = json.loads(ai_hunter_config_str)
main_config['ai_hunter_config'] = ai_hunter_config
print("🤖 AI Hunter: Loaded configuration from environment")
except json.JSONDecodeError:
print("⚠️ AI Hunter: Failed to parse AI_HUNTER_CONFIG from environment")
# If no AI Hunter config in environment, try to load from file as fallback
if 'ai_hunter_config' not in main_config:
# Try multiple locations for config.json
config_paths = [
os.path.join(os.getcwd(), 'config.json'),
os.path.join(out, '..', 'config.json'),
]
if getattr(sys, 'frozen', False):
config_paths.append(os.path.join(os.path.dirname(sys.executable), 'config.json'))
else:
script_dir = os.path.dirname(os.path.abspath(__file__))
config_paths.extend([
os.path.join(script_dir, 'config.json'),
os.path.join(os.path.dirname(script_dir), 'config.json')
])
for config_path in config_paths:
if os.path.exists(config_path):
try:
with open(config_path, 'r', encoding='utf-8') as f:
file_config = json.load(f)
if 'ai_hunter_config' in file_config:
main_config['ai_hunter_config'] = file_config['ai_hunter_config']
print(f"🤖 AI Hunter: Loaded configuration from {config_path}")
break
except Exception as e:
print(f"⚠️ Failed to load config from {config_path}: {e}")
# Always create and inject the improved AI Hunter when ai-hunter mode is selected
ai_hunter = ImprovedAIHunterDetection(main_config)
# The TranslationProcessor class has a method that checks for duplicates
# We need to replace it with our enhanced AI Hunter
# Create a wrapper to match the expected signature
def enhanced_duplicate_check(self, result, idx, prog, out, actual_num=None):
# If actual_num is not provided, try to get it from progress
if actual_num is None:
# Look for the chapter being processed
for ch_key, ch_info in prog.get("chapters", {}).items():
if ch_info.get("chapter_idx") == idx:
actual_num = ch_info.get("actual_num", idx + 1)
break
# Fallback to idx+1 if not found
if actual_num is None:
actual_num = idx + 1
return ai_hunter.detect_duplicate_ai_hunter_enhanced(result, idx, prog, out, actual_num)
# Bind the enhanced method to the processor instance
translation_processor.check_duplicate_content = enhanced_duplicate_check.__get__(translation_processor, TranslationProcessor)
print("🤖 AI Hunter: Using enhanced detection with configurable thresholds")
# First pass: set actual chapter numbers respecting the config
for idx, c in enumerate(chapters):
raw_num = FileUtilities.extract_actual_chapter_number(c, patterns=None, config=config)
#print(f"[DEBUG] Extracted raw_num={raw_num} from {c.get('original_basename', 'unknown')}")
# Apply offset if configured
offset = config.CHAPTER_NUMBER_OFFSET if hasattr(config, 'CHAPTER_NUMBER_OFFSET') else 0
raw_num += offset
if config.DISABLE_ZERO_DETECTION:
# Use raw numbers without adjustment
c['actual_chapter_num'] = raw_num
c['raw_chapter_num'] = raw_num
c['zero_adjusted'] = False
else:
# Store raw number
c['raw_chapter_num'] = raw_num
# Apply 0-based adjustment if detected
if uses_zero_based:
c['actual_chapter_num'] = raw_num + 1
c['zero_adjusted'] = True
else:
c['actual_chapter_num'] = raw_num
c['zero_adjusted'] = False
# Second pass: process chapters
for idx, c in enumerate(chapters):
chap_num = c["num"]
# Check if this is a pre-split text chunk with decimal number
if (is_text_file and c.get('is_chunk', False) and isinstance(c['num'], float)):
actual_num = c['num'] # Preserve the decimal for text files only
else:
actual_num = c.get('actual_chapter_num', c['num'])
content_hash = c.get("content_hash") or ContentProcessor.get_content_hash(c["body"])
if start is not None and not (start <= actual_num <= end):
#print(f"[SKIP] Chapter {actual_num} (file: {c.get('original_basename', 'unknown')}) outside range {start}-{end}")
continue
needs_translation, skip_reason, existing_file = progress_manager.check_chapter_status(
idx, actual_num, content_hash, out, c # Pass the chapter object
)
# Add explicit file check for supposedly completed chapters
if not needs_translation and existing_file:
file_path = os.path.join(out, existing_file)
if not os.path.exists(file_path):
print(f"⚠️ Output file missing for chapter {actual_num}: {existing_file}")
needs_translation = True
skip_reason = None
# Update status to file_missing
progress_manager.update(idx, actual_num, content_hash, None, status="file_missing")
progress_manager.save()
if not needs_translation:
# Modify skip_reason to use appropriate terminology
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
# Replace "Chapter" with appropriate terminology in skip_reason
skip_reason_modified = skip_reason.replace("Chapter", terminology)
print(f"[SKIP] {skip_reason_modified}")
continue
chapter_position = f"{chapters_completed + 1}/{chapters_to_process}"
# Determine if this is a text file
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
# Determine file reference based on type
if c.get('is_chunk', False):
file_ref = f"Section_{c['num']}"
else:
file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
print(f"\n🔄 Processing #{idx+1}/{total_chapters} (Actual: {terminology} {actual_num}) ({chapter_position} to translate): {c['title']} [File: {file_ref}]")
chunk_context_manager.start_chapter(chap_num, c['title'])
has_images = c.get('has_images', False)
has_meaningful_text = ContentProcessor.is_meaningful_text_content(c["body"])
text_size = c.get('file_size', 0)
is_empty_chapter = (not has_images and text_size < 10)
is_image_only_chapter = (has_images and not has_meaningful_text)
is_mixed_content = (has_images and has_meaningful_text)
is_text_only = (not has_images and has_meaningful_text)
if is_empty_chapter:
print(f"📄 Empty chapter {actual_num} detected")
# Create filename for empty chapter
if isinstance(c['num'], float):
fname = FileUtilities.create_chapter_filename(c, c['num'])
else:
fname = FileUtilities.create_chapter_filename(c, actual_num)
# Save original content
with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
f.write(c["body"])
# Update progress tracking
progress_manager.update(idx, actual_num, content_hash, fname, status="completed_empty")
progress_manager.save()
chapters_completed += 1
# CRITICAL: Skip translation!
continue
elif is_image_only_chapter:
print(f"📸 Image-only chapter: {c.get('image_count', 0)} images")
translated_html = c["body"]
image_translations = {}
# Step 1: Process images if image translation is enabled
if image_translator and config.ENABLE_IMAGE_TRANSLATION:
print(f"🖼️ Translating {c.get('image_count', 0)} images...")
image_translator.set_current_chapter(chap_num)
translated_html, image_translations = process_chapter_images(
c["body"],
actual_num,
image_translator,
check_stop
)
if image_translations:
print(f"✅ Translated {len(image_translations)} images")
# Step 2: Check for headers/titles that need translation
from bs4 import BeautifulSoup
soup = BeautifulSoup(c["body"], 'html.parser')
# Look for headers
headers = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
# If we have headers, we should translate them even in "image-only" chapters
if headers and any(h.get_text(strip=True) for h in headers):
print(f"📝 Found headers to translate in image-only chapter")
# Create a minimal HTML with just the headers for translation
headers_html = ""
for header in headers:
if header.get_text(strip=True):
headers_html += str(header) + "\n"
if headers_html:
print(f"📤 Translating chapter headers...")
# Send just the headers for translation
header_msgs = base_msg + [{"role": "user", "content": headers_html}]
# Use the standard filename
fname = FileUtilities.create_chapter_filename(c, actual_num)
client.set_output_filename(fname)
# Simple API call for headers
header_result, _ = client.send(
header_msgs,
temperature=config.TEMP,
max_tokens=config.MAX_OUTPUT_TOKENS
)
if header_result:
# Clean the result
header_result = re.sub(r"^```(?:html)?\s*\n?", "", header_result, count=1, flags=re.MULTILINE)
header_result = re.sub(r"\n?```\s*$", "", header_result, count=1, flags=re.MULTILINE)
# Parse both the translated headers and the original body
soup_headers = BeautifulSoup(header_result, 'html.parser')
soup_body = BeautifulSoup(translated_html, 'html.parser')
# Replace headers in the body with translated versions
translated_headers = soup_headers.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
original_headers = soup_body.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'title'])
# Match and replace headers
for orig, trans in zip(original_headers, translated_headers):
if trans and trans.get_text(strip=True):
orig.string = trans.get_text(strip=True)
translated_html = str(soup_body)
print(f"✅ Headers translated successfully")
status = "completed"
else:
print(f"⚠️ Failed to translate headers")
status = "completed_image_only"
else:
status = "completed_image_only"
else:
print(f"ℹ️ No headers found to translate")
status = "completed_image_only"
# Step 3: Save with correct filename
fname = FileUtilities.create_chapter_filename(c, actual_num)
with open(os.path.join(out, fname), 'w', encoding='utf-8') as f:
f.write(translated_html)
print(f"[Chapter {idx+1}/{total_chapters}] ✅ Saved image-only chapter")
progress_manager.update(idx, actual_num, content_hash, fname, status=status)
progress_manager.save()
chapters_completed += 1
continue
else:
# Set default text to translate
text_to_translate = c["body"]
image_translations = {}
if is_mixed_content and image_translator and config.ENABLE_IMAGE_TRANSLATION:
print(f"🖼️ Processing {c.get('image_count', 0)} images first...")
print(f"[DEBUG] Content before image processing (first 200 chars):")
print(c["body"][:200])
print(f"[DEBUG] Has h1 tags: {'' in c['body']}")
print(f"[DEBUG] Has h2 tags: {'' in c['body']}")
image_translator.set_current_chapter(chap_num)
# Store the original body before processing
original_body = c["body"]
# Calculate original chapter tokens before modification
original_chapter_tokens = chapter_splitter.count_tokens(original_body)
# Process images and get body with translations
body_with_images, image_translations = process_chapter_images(
c["body"],
actual_num,
image_translator,
check_stop
)
if image_translations:
print(f"✅ Translated {len(image_translations)} images")
# Store the body with images for later merging
c["body_with_images"] = c["body"]
# For chapters with only images and title, we still need to translate the title
# Extract clean text for translation from ORIGINAL body
from bs4 import BeautifulSoup
soup_clean = BeautifulSoup(original_body, 'html.parser')
# Remove images from the original to get pure text
for img in soup_clean.find_all('img'):
img.decompose()
# Set clean text for translation - use prettify() or str() on the full document
c["body"] = str(soup_clean) if soup_clean.body else original_body
# If there's no meaningful text content after removing images,
# the text translation will just translate the title, which is correct
print(f" 📝 Clean text for translation: {len(c['body'])} chars")
# Update text_size to reflect actual text to translate
text_size = len(c["body"])
# Recalculate the actual token count for clean text
actual_text_tokens = chapter_splitter.count_tokens(c["body"])
print(f" 📊 Actual text tokens: {actual_text_tokens} (was counting {original_chapter_tokens} with images)")
else:
print(f"ℹ️ No translatable text found in images")
# Keep original body if no image translations
c["body"] = original_body
print(f"📖 Translating text content ({text_size} characters)")
progress_manager.update(idx, actual_num, content_hash, output_file=None, status="in_progress")
progress_manager.save()
# Apply ignore filtering to the content before chunk splitting
batch_translate_active = os.getenv('BATCH_TRANSLATE_HEADERS', '0') == '1'
ignore_title_tag = os.getenv('IGNORE_TITLE', '0') == '1' and batch_translate_active
ignore_header_tags = os.getenv('IGNORE_HEADER', '0') == '1' and batch_translate_active
if (ignore_title_tag or ignore_header_tags) and c["body"]:
from bs4 import BeautifulSoup
content_soup = BeautifulSoup(c["body"], 'html.parser')
# Remove title tags if ignored
if ignore_title_tag:
for title_tag in content_soup.find_all('title'):
title_tag.decompose()
# Remove header tags if ignored
if ignore_header_tags:
for header_tag in content_soup.find_all(['h1', 'h2', 'h3']):
header_tag.decompose()
c["body"] = str(content_soup) # Update the chapter body
# Check if this chapter is already a chunk from text file splitting
if c.get('is_chunk', False):
# This is already a pre-split chunk, but still check if it needs further splitting
# Calculate based on OUTPUT limit only
max_output_tokens = config.MAX_OUTPUT_TOKENS
safety_margin_output = 500
# CJK to English typically compresses to 0.7-0.9x
compression_factor = config.COMPRESSION_FACTOR
available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
# Ensure minimum
available_tokens = max(available_tokens, 1000)
print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
chapter_tokens = chapter_splitter.count_tokens(c["body"])
if chapter_tokens > available_tokens:
# Even pre-split chunks might need further splitting
chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
print(f"📄 Section {c['num']} (pre-split from text file) needs further splitting into {len(chunks)} chunks")
else:
chunks = [(c["body"], 1, 1)]
print(f"📄 Section {c['num']} (pre-split from text file)")
else:
# Normal splitting logic for non-text files
# Calculate based on OUTPUT limit only
max_output_tokens = config.MAX_OUTPUT_TOKENS
safety_margin_output = 500
# CJK to English typically compresses to 0.7-0.9x
compression_factor = config.COMPRESSION_FACTOR
available_tokens = int((max_output_tokens - safety_margin_output) / compression_factor)
# Ensure minimum
available_tokens = max(available_tokens, 1000)
print(f"📊 Chunk size: {available_tokens:,} tokens (based on {max_output_tokens:,} output limit, compression: {compression_factor})")
chunks = chapter_splitter.split_chapter(c["body"], available_tokens)
# Use consistent terminology
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
print(f"📄 {terminology} will be processed in {len(chunks)} chunk(s)")
# Recalculate tokens on the actual text to be translated
actual_chapter_tokens = chapter_splitter.count_tokens(c["body"])
if len(chunks) > 1:
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (limit: {available_tokens:,} tokens per chunk)")
else:
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
print(f" ℹ️ {terminology} size: {actual_chapter_tokens:,} tokens (within limit of {available_tokens:,} tokens)")
chapter_key_str = str(idx)
if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
progress_manager.prog["chapter_chunks"][chapter_key_str] = {
"total": len(chunks),
"completed": [],
"chunks": {}
}
progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
translated_chunks = []
for chunk_html, chunk_idx, total_chunks in chunks:
chapter_key_str = content_hash
old_key_str = str(idx)
if chapter_key_str not in progress_manager.prog.get("chapter_chunks", {}) and old_key_str in progress_manager.prog.get("chapter_chunks", {}):
progress_manager.prog["chapter_chunks"][chapter_key_str] = progress_manager.prog["chapter_chunks"][old_key_str]
del progress_manager.prog["chapter_chunks"][old_key_str]
#print(f"[PROGRESS] Migrated chunks for chapter {chap_num} to new tracking system")
if chapter_key_str not in progress_manager.prog["chapter_chunks"]:
progress_manager.prog["chapter_chunks"][chapter_key_str] = {
"total": len(chunks),
"completed": [],
"chunks": {}
}
progress_manager.prog["chapter_chunks"][chapter_key_str]["total"] = len(chunks)
# Get chapter status to check for qa_failed
chapter_info = progress_manager.prog["chapters"].get(chapter_key_str, {})
chapter_status = chapter_info.get("status")
if chapter_status == "qa_failed":
# Force retranslation of qa_failed chapters
print(f" [RETRY] Chunk {chunk_idx}/{total_chunks} - retranslating due to QA failure")
if config.CONTEXTUAL and history_manager.will_reset_on_next_append(config.HIST_LIMIT):
print(f" 📌 History will reset after this chunk (current: {len(history_manager.load_history())//2}/{config.HIST_LIMIT} exchanges)")
if check_stop():
print(f"❌ Translation stopped during chapter {actual_num}, chunk {chunk_idx}")
return
current_chunk_number += 1
progress_percent = (current_chunk_number / total_chunks_needed) * 100 if total_chunks_needed > 0 else 0
if chunks_completed > 0:
elapsed_time = time.time() - translation_start_time
avg_time_per_chunk = elapsed_time / chunks_completed
remaining_chunks = total_chunks_needed - current_chunk_number + 1
eta_seconds = remaining_chunks * avg_time_per_chunk
eta_hours = int(eta_seconds // 3600)
eta_minutes = int((eta_seconds % 3600) // 60)
eta_str = f"{eta_hours}h {eta_minutes}m" if eta_hours > 0 else f"{eta_minutes}m"
else:
eta_str = "calculating..."
if total_chunks > 1:
print(f" 🔄 Translating chunk {chunk_idx}/{total_chunks} for #{idx+1} (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str})")
print(f" ⏳ Chunk size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
else:
# Determine terminology and file reference
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
# Consistent file reference
if c.get('is_chunk', False):
file_ref = f"Section_{c['num']}"
else:
file_ref = c.get('original_basename', f'{terminology}_{actual_num}')
print(f" 📄 Translating {terminology.lower()} content (Overall: {current_chunk_number}/{total_chunks_needed} - {progress_percent:.1f}% - ETA: {eta_str}) [File: {file_ref}]")
print(f" 📊 {terminology} {actual_num} size: {len(chunk_html):,} characters (~{chapter_splitter.count_tokens(chunk_html):,} tokens)")
print(f" ℹ️ This may take 30-60 seconds. Stop will take effect after completion.")
if log_callback:
if hasattr(log_callback, '__self__') and hasattr(log_callback.__self__, 'append_chunk_progress'):
if total_chunks == 1:
# Determine terminology based on source type
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
log_callback.__self__.append_chunk_progress(
1, 1, "text",
f"{terminology} {actual_num}",
overall_current=current_chunk_number,
overall_total=total_chunks_needed,
extra_info=f"{len(chunk_html):,} chars"
)
else:
log_callback.__self__.append_chunk_progress(
chunk_idx,
total_chunks,
"text",
f"{terminology} {actual_num}",
overall_current=current_chunk_number,
overall_total=total_chunks_needed
)
else:
# Determine terminology based on source type
is_text_source = is_text_file or c.get('filename', '').endswith('.txt') or c.get('is_chunk', False)
terminology = "Section" if is_text_source else "Chapter"
terminology_lower = "section" if is_text_source else "chapter"
if total_chunks == 1:
log_callback(f"📄 Processing {terminology} {actual_num} ({chapters_completed + 1}/{chapters_to_process}) - {progress_percent:.1f}% complete")
else:
log_callback(f"📄 processing chunk {chunk_idx}/{total_chunks} for {terminology_lower} {actual_num} - {progress_percent:.1f}% complete")
# Get custom chunk prompt template from environment
chunk_prompt_template = os.getenv("TRANSLATION_CHUNK_PROMPT", "[PART {chunk_idx}/{total_chunks}]\n{chunk_html}")
if total_chunks > 1:
user_prompt = chunk_prompt_template.format(
chunk_idx=chunk_idx,
total_chunks=total_chunks,
chunk_html=chunk_html
)
else:
user_prompt = chunk_html
if config.CONTEXTUAL:
history = history_manager.load_history()
trimmed = history[-config.HIST_LIMIT*2:]
chunk_context = chunk_context_manager.get_context_messages(limit=2)
else:
history = [] # Set empty history when not contextual
trimmed = []
chunk_context = []
# Build the current system prompt from the original each time, and append the last summary block if present
current_system_content = original_system_prompt
if config.USE_ROLLING_SUMMARY and last_summary_block_text:
current_system_content = (
current_system_content
+ "\n\n[Rolling Summary of Previous Chapter]\n"
+ "(For AI: Use as context only; do not include in output)\n"
+ last_summary_block_text
+ "\n[End of Rolling Summary]"
)
current_base = [{"role": "system", "content": current_system_content}]
# If we have a prepared rolling summary from previous chapter, include it as a separate message (do NOT mutate system prompt)
summary_msgs_list = []
if config.USE_ROLLING_SUMMARY and last_summary_block_text:
summary_msgs_list = [{
"role": os.getenv("SUMMARY_ROLE", "user"),
"content": (
"CONTEXT ONLY - DO NOT INCLUDE IN TRANSLATION:\n"
"[MEMORY] Previous context summary:\n\n"
f"{last_summary_block_text}\n\n"
"[END MEMORY]\n"
"END OF CONTEXT - BEGIN ACTUAL CONTENT TO TRANSLATE:"
)
}]
msgs = current_base + summary_msgs_list + chunk_context + trimmed + [{"role": "user", "content": user_prompt}]
c['__index'] = idx
c['__progress'] = progress_manager.prog
c['history_manager'] = history_manager
result, finish_reason = translation_processor.translate_with_retry(
msgs, chunk_html, c, chunk_idx, total_chunks
)
if result is None:
progress_manager.update(idx, actual_num, content_hash, output_file=None, status="failed")
progress_manager.save()
continue
if config.REMOVE_AI_ARTIFACTS:
result = ContentProcessor.clean_ai_artifacts(result, True)
if config.EMERGENCY_RESTORE:
result = ContentProcessor.emergency_restore_paragraphs(result, chunk_html)
if config.REMOVE_AI_ARTIFACTS:
lines = result.split('\n')
json_line_count = 0
for i, line in enumerate(lines[:5]):
if line.strip() and any(pattern in line for pattern in [
'"role":', '"content":', '"messages":',
'{"role"', '{"content"', '[{', '}]'
]):
json_line_count = i + 1
else:
break
if json_line_count > 0 and json_line_count < len(lines):
remaining = '\n'.join(lines[json_line_count:])
if remaining.strip() and len(remaining) > 100:
result = remaining
print(f"✂️ Removed {json_line_count} lines of JSON artifacts")
result = re.sub(r'\[PART \d+/\d+\]\s*', '', result, flags=re.IGNORECASE)
translated_chunks.append((result, chunk_idx, total_chunks))
chunk_context_manager.add_chunk(user_prompt, result, chunk_idx, total_chunks)
progress_manager.prog["chapter_chunks"][chapter_key_str]["completed"].append(chunk_idx)
progress_manager.prog["chapter_chunks"][chapter_key_str]["chunks"][str(chunk_idx)] = result
progress_manager.save()
chunks_completed += 1
will_reset = history_manager.will_reset_on_next_append(
config.HIST_LIMIT if config.CONTEXTUAL else 0,
config.TRANSLATION_HISTORY_ROLLING
)
history = history_manager.append_to_history(
user_prompt,
result,
config.HIST_LIMIT if config.CONTEXTUAL else 0,
reset_on_limit=True,
rolling_window=config.TRANSLATION_HISTORY_ROLLING
)
if chunk_idx < total_chunks:
# Handle float delays while checking for stop
full_seconds = int(config.DELAY)
fractional_second = config.DELAY - full_seconds
# Check stop signal every second for full seconds
for i in range(full_seconds):
if check_stop():
print("❌ Translation stopped during delay")
return
time.sleep(1)
# Handle the fractional part if any
if fractional_second > 0:
if check_stop():
print("❌ Translation stopped during delay")
return
time.sleep(fractional_second)
if check_stop():
print(f"❌ Translation stopped before saving chapter {actual_num}")
return
if len(translated_chunks) > 1:
print(f" 📎 Merging {len(translated_chunks)} chunks...")
translated_chunks.sort(key=lambda x: x[1])
merged_result = chapter_splitter.merge_translated_chunks(translated_chunks)
else:
merged_result = translated_chunks[0][0] if translated_chunks else ""
if config.CONTEXTUAL and len(translated_chunks) > 1:
user_summary, assistant_summary = chunk_context_manager.get_summary_for_history()
if user_summary and assistant_summary:
history_manager.append_to_history(
user_summary,
assistant_summary,
config.HIST_LIMIT,
reset_on_limit=False,
rolling_window=config.TRANSLATION_HISTORY_ROLLING
)
print(f" 📝 Added chapter summary to history")
chunk_context_manager.clear()
# For text file chunks, ensure we pass the decimal number
if is_text_file and c.get('is_chunk', False) and isinstance(c.get('num'), float):
fname = FileUtilities.create_chapter_filename(c, c['num']) # Use the decimal num directly
else:
fname = FileUtilities.create_chapter_filename(c, actual_num)
client.set_output_filename(fname)
cleaned = re.sub(r"^```(?:html)?\s*\n?", "", merged_result, count=1, flags=re.MULTILINE)
cleaned = re.sub(r"\n?```\s*$", "", cleaned, count=1, flags=re.MULTILINE)
cleaned = ContentProcessor.clean_ai_artifacts(cleaned, remove_artifacts=config.REMOVE_AI_ARTIFACTS)
if is_mixed_content and image_translations:
print(f"🔀 Merging {len(image_translations)} image translations with text...")
from bs4 import BeautifulSoup
# Parse the translated text (which has the translated title/header)
soup_translated = BeautifulSoup(cleaned, 'html.parser')
# For each image translation, insert it into the document
for img_path, translation_html in image_translations.items():
if translation_html and ' 0:
combined.write(f"\n\n{'='*50}\n\n")
# Write the original chapter title (without Part X/Y suffix)
original_title = chapter_data['title']
# Remove the (Part X/Y) suffix if present
if ' (Part ' in original_title:
original_title = original_title.split(' (Part ')[0]
combined.write(f"{original_title}\n\n")
# Add the chunk content
combined.write(content)
# Add spacing between chunks of the same chapter
if chunk_idx < total_chunks:
combined.write("\n\n")
else:
# This is a standalone chapter
current_main_chapter = chapter_data['num']
# Add separator if not first chapter
if i > 0:
combined.write(f"\n\n{'='*50}\n\n")
# Write the chapter title
combined.write(f"{chapter_data['title']}\n\n")
# Add the content
combined.write(content)
print(f" • Combined file with preserved sections: {combined_path}")
total_time = time.time() - translation_start_time
hours = int(total_time // 3600)
minutes = int((total_time % 3600) // 60)
seconds = int(total_time % 60)
print(f"\n⏱️ Total translation time: {hours}h {minutes}m {seconds}s")
print(f"📊 Chapters completed: {chapters_completed}")
print(f"✅ Text file translation complete!")
if log_callback:
log_callback(f"✅ Text file translation complete! Created {combined_path}")
except Exception as e:
print(f"❌ Error creating combined text file: {e}")
if log_callback:
log_callback(f"❌ Error creating combined text file: {e}")
else:
print("🔍 Checking for translated chapters...")
# Respect retain extension toggle: if enabled, don't look for response_ prefix
if should_retain_source_extension():
response_files = [f for f in os.listdir(out) if f.endswith('.html') and not f.startswith('chapter_')]
else:
response_files = [f for f in os.listdir(out) if f.startswith('response_') and f.endswith('.html')]
chapter_files = [f for f in os.listdir(out) if f.startswith('chapter_') and f.endswith('.html')]
if not response_files and chapter_files:
if should_retain_source_extension():
print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
print("ℹ️ Retain-source-extension mode is ON: skipping placeholder creation and using original files for EPUB compilation.")
else:
print(f"⚠️ No translated files found, but {len(chapter_files)} original chapters exist")
print("📝 Creating placeholder response files for EPUB compilation...")
for chapter_file in chapter_files:
response_file = chapter_file.replace('chapter_', 'response_', 1)
src = os.path.join(out, chapter_file)
dst = os.path.join(out, response_file)
try:
with open(src, 'r', encoding='utf-8') as f:
content = f.read()
soup = BeautifulSoup(content, 'html.parser')
notice = soup.new_tag('p')
notice.string = "[Note: This chapter could not be translated - showing original content]"
notice['style'] = "color: red; font-style: italic;"
if soup.body:
soup.body.insert(0, notice)
with open(dst, 'w', encoding='utf-8') as f:
f.write(str(soup))
except Exception as e:
print(f"⚠️ Error processing {chapter_file}: {e}")
try:
shutil.copy2(src, dst)
except:
pass
print(f"✅ Created {len(chapter_files)} placeholder response files")
print("⚠️ Note: The EPUB will contain untranslated content")
print("📘 Building final EPUB…")
try:
from epub_converter import fallback_compile_epub
fallback_compile_epub(out, log_callback=log_callback)
print("✅ All done: your final EPUB is in", out)
total_time = time.time() - translation_start_time
hours = int(total_time // 3600)
minutes = int((total_time % 3600) // 60)
seconds = int(total_time % 60)
print(f"\n📊 Translation Statistics:")
print(f" • Total chunks processed: {chunks_completed}")
print(f" • Total time: {hours}h {minutes}m {seconds}s")
if chunks_completed > 0:
avg_time = total_time / chunks_completed
print(f" • Average time per chunk: {avg_time:.1f} seconds")
stats = progress_manager.get_stats(out)
print(f"\n📊 Progress Tracking Summary:")
print(f" • Total chapters tracked: {stats['total_tracked']}")
print(f" • Successfully completed: {stats['completed']}")
print(f" • Missing files: {stats['missing_files']}")
print(f" • In progress: {stats['in_progress']}")
except Exception as e:
print("❌ EPUB build failed:", e)
print("TRANSLATION_COMPLETE_SIGNAL")
if __name__ == "__main__":
main()